glustercli.volume_parser source code

1 module glustercli.volume_parser;
2 
3 import std.process;
4 import std..string;
5 import std.conv;
6 import std.json;
7 import std.algorithm: map;
8 import std.array: array;
9 
10 import arsd.dom;
11 
12 enum VolumeTypeDist = "Distribute";
13 enum VolumeTypeRep = "Replicate";
14 enum VolumeTypeDisp = "Disperse";
15 enum VolumeTypeDistRep = "Distributed Replicate";
16 enum VolumeTypeDistDisp = "Distributed Disperse";
17 
18 enum SubvolTypeDist = "Distribute";
19 enum SubvolTypeRep = "Replicate";
20 enum SubvolTypeDisp = "Disperse";
21 
22 enum BrickTypeDefault = "Brick";
23 enum BrickTypeArbiter = "Arbiter";
24 
25 enum HealthUp = "up";
26 enum HealthDown = "down";
27 enum HealthPartial = "partial";
28 enum HealthDegraded = "degraded";
29 
30 enum StateCreated = "Created";
31 enum StateStarted = "Started";
32 enum StateStopped = "Stopped";
33 
34 struct Volume
35 {
36     string name;
37     string type;
38     string state;
39     string health;
40     string id;
41     int numSubvols;
42     int numBricks;
43     int replicaCount;
44     int arbiterCount;
45     int disperseCount;
46     int disperseRedundancyCount;
47     string transport;
48     SubVolume[] subvols;
49     ulong sizeTotal;
50     ulong sizeUsed;
51     ulong inodesTotal;
52     ulong inodesUsed;
53     Option[] options;
54     JSONValue toJson()
55     {
56         return JSONValue(
57             [
58                 "name": JSONValue(name),
59                 "type": JSONValue(type),
60                 "state": JSONValue(state),
61                 "health": JSONValue(health),
62                 "id": JSONValue(id),
63                 "num_subvols": JSONValue(numSubvols),
64                 "num_bricks": JSONValue(numBricks),
65                 "replica_count": JSONValue(replicaCount),
66                 "arbiter_count": JSONValue(arbiterCount),
67                 "disperse_count": JSONValue(disperseCount),
68                 "disperse_redundancy_count": JSONValue(disperseRedundancyCount),
69                 "transport": JSONValue(transport),
70                 "subvols": JSONValue(subvols.map!(subvol => subvol.toJson).array),
71                 "size_total": JSONValue(sizeTotal),
72                 "size_used": JSONValue(sizeUsed),
73                 "inodes_total": JSONValue(inodesTotal),
74                 "inodes_used": JSONValue(inodesUsed),
75                 "options": JSONValue(options.map!(opt => opt.toJson).array),
76                 ]
77             );
78     }
79 }
80 
81 struct Option
82 {
83     string name;
84     string value;
85     string volumeId;
86     JSONValue toJson()
87     {
88         return JSONValue(
89             [
90                 "name": JSONValue(name),
91                 "value": JSONValue(value),
92                 "volumes_id": JSONValue(volumeId)
93             ]
94         );
95     }
96 }
97 
98 struct SubVolume
99 {
100     string id;
101     string health;
102     int replicaCount;
103     int arbiterCount;
104     int disperseCount;
105     int disperseRedundancyCount;
106     string type;
107     Brick[] bricks;
108     string volumeId;
109     ulong numBricks;
110     JSONValue toJson()
111     {
112         return JSONValue(
113             [
114                 "id": JSONValue(id),
115                 "health": JSONValue(health),
116                 "replica_count": JSONValue(replicaCount),
117                 "arbiter_count": JSONValue(arbiterCount),
118                 "disperse_count": JSONValue(disperseCount),
119                 "disperse_redundancy_count": JSONValue(disperseRedundancyCount),
120                 "type": JSONValue(type),
121                 "bricks": JSONValue(bricks.map!(brick => brick.toJson).array),
122                 "num_bricks": JSONValue(numBricks),
123                 "volumes_id": JSONValue(volumeId)
124             ]
125         );
126     }
127 }
128 
129 struct Brick
130 {
131     string host;
132     string path;
133     string nodeid;
134     string state;
135     string type;
136     int port;
137     int pid;
138     ulong sizeTotal;
139     ulong sizeUsed;
140     ulong inodesTotal;
141     ulong inodesUsed;
142     string fsType;
143     string device;
144     int blockSize;
145     string mountOptions;
146     string volumeId;
147     string subvolId;
148     JSONValue toJson()
149     {
150         return JSONValue(
151             [
152                 "host": JSONValue(host),
153                 "path": JSONValue(path),
154                 "peers_id": JSONValue(nodeid),
155                 "state": JSONValue(state),
156                 "type": JSONValue(type),
157                 "port": JSONValue(port),
158                 "pid": JSONValue(pid),
159                 "size_total": JSONValue(sizeTotal),
160                 "size_used": JSONValue(sizeUsed),
161                 "inodes_total": JSONValue(inodesTotal),
162                 "inodes_used": JSONValue(inodesUsed),
163                 "fs": JSONValue(fsType),
164                 "device": JSONValue(device),
165                 "block_size": JSONValue(blockSize),
166                 "mount_options": JSONValue(mountOptions),
167                 "volumes_id": JSONValue(volumeId),
168                 "subvols_id": JSONValue(subvolId),
169             ]
170         );
171     }
172 }
173 
174 string transportType(string val)
175 {
176     if (val == "0")
177         return "TCP";
178     else if (val == "1")
179         return "RDMA";
180 
181     return "TCP,RDMA";
182 }
183 
184 int parsePortOrPid(string val)
185 {
186     try
187     {
188         return to!int(val);
189     }
190     catch (ConvException)
191     {
192         return -1;
193     }
194 }
195 
196 Brick[string] parseBricksFromVolStatus(Element[] elements)
197 {
198     Brick[string] bricks;
199     foreach(ele; elements)
200     {
201         auto brick = Brick();
202         brick.host = ele.querySelector("hostname").innerText;
203         brick.path = ele.querySelector("path").innerText;
204         brick.nodeid = ele.querySelector("peerid").innerText;
205         brick.fsType = ele.querySelector("fsName").innerText;
206         brick.device = ele.querySelector("device").innerText;
207         brick.blockSize = to!int(ele.querySelector("blockSize").innerText);
208         brick.mountOptions = ele.querySelector("mntOptions").innerText;
209         brick.state = (ele.querySelector("status").innerText == "1" ? HealthUp : HealthDown);
210         brick.sizeTotal = to!ulong(ele.querySelector("sizeTotal").innerText);
211         brick.sizeUsed = brick.sizeTotal - to!ulong(ele.querySelector("sizeFree").innerText);
212         brick.inodesTotal = to!ulong(ele.querySelector("inodesTotal").innerText);
213         brick.inodesUsed = brick.inodesTotal - to!ulong(ele.querySelector("inodesFree").innerText);
214         brick.port = parsePortOrPid(ele.querySelector("port").innerText);
215         brick.pid = parsePortOrPid(ele.querySelector("pid").innerText);
216 
217         bricks[brick.host ~ ":" ~ brick.path] = brick;
218     }
219     return bricks;
220 }
221 
222 
223 Option[] parseOptionsFromVolinfo(Volume vol, Element[] elements)
224 {
225     Option[] options;
226     foreach(ele; elements)
227     {
228         options ~= Option(
229             ele.querySelector("name").innerText,
230             ele.querySelector("value").innerText,
231             vol.id
232             );
233     }
234     return options;
235 }
236 
237 Brick[] parseBricksFromVolinfo(Element[] elements)
238 {
239     Brick[] bricks;
240     foreach(ele; elements)
241     {
242         auto brick = Brick();
243         auto hostAndPath = ele.querySelector("name").innerText.split(":");
244         brick.path = hostAndPath[1];
245         brick.host = hostAndPath[0];
246         brick.nodeid = ele.querySelector("hostUuid").innerText;
247         brick.type = BrickTypeDefault;
248         brick.type = (ele.querySelector("isArbiter").innerText == "1" ? BrickTypeArbiter : BrickTypeDefault);
249 
250         brick.blockSize = 4096;
251         brick.state = "Unknown";
252         bricks ~= brick;
253     }
254     return bricks;
255 }
256 
257 int getSubvolBricksCount(int replicaCount, int disperseCount)
258 {
259 	if (replicaCount > 0)
260 		return replicaCount;
261 
262     if (disperseCount > 0)
263 		return disperseCount;
264 
265 	return 1;
266 }
267 
268 
269 string getSubvolType(string voltype)
270 {
271 	switch (voltype)
272     {
273 	case VolumeTypeDistRep:
274 		return SubvolTypeRep;
275 	case VolumeTypeDistDisp:
276 		return SubvolTypeDisp;
277 	default:
278 		return voltype;
279 	}
280 }
281 
282 void parseVolumesFromVolinfo(string[] output, ref Volume[] volumes)
283 {
284     auto document = new Document(output.join(""));
285     auto elements = document.getElementsByTagName("volume");
286     foreach(ele; elements)
287     {
288         auto vol = Volume();
289         vol.health = HealthDown;
290         vol.name = ele.querySelector("name").innerText;
291         vol.type = ele.querySelector("typeStr").innerText.replace("-", " ");
292         vol.state = ele.querySelector("statusStr").innerText;
293         vol.id = ele.querySelector("id").innerText;
294         vol.numBricks = to!int(ele.querySelector("brickCount").innerText);
295         vol.replicaCount = to!int(ele.querySelector("replicaCount").innerText);
296         vol.disperseCount = to!int(ele.querySelector("disperseCount").innerText);
297         vol.arbiterCount = to!int(ele.querySelector("arbiterCount").innerText);
298         vol.disperseRedundancyCount = to!int(ele.querySelector("redundancyCount").innerText);
299         vol.transport = transportType(ele.querySelector("transport").innerText);
300 
301         vol.options = parseOptionsFromVolinfo(vol, ele.getElementsByTagName("option"));
302         
303         // Parse the bricks and create subvol groups
304         auto bricks = parseBricksFromVolinfo(ele.getElementsByTagName("brick"));
305         auto subvolBricksCount = getSubvolBricksCount(vol.replicaCount, vol.disperseCount);
306         vol.numSubvols = vol.numBricks / subvolBricksCount;
307         
308         foreach(sidx; 0 .. vol.numSubvols)
309         {
310             auto subvol = SubVolume();
311             subvol.volumeId = vol.id;
312             subvol.health = HealthDown;
313             subvol.type = getSubvolType(vol.type);
314             subvol.replicaCount = vol.replicaCount;
315             subvol.arbiterCount = vol.arbiterCount;
316             subvol.disperseCount = vol.disperseCount;
317             subvol.disperseRedundancyCount = vol.disperseRedundancyCount;
318             subvol.id = format!"%s-%s-%d"(vol.name, subvol.type.toLower, sidx);
319             foreach(bidx; 0 .. subvolBricksCount)
320             {
321                 subvol.bricks ~= bricks[sidx+bidx];
322             }
323             subvol.numBricks = bricks.length;
324             vol.subvols ~= subvol;
325         }
326        
327         volumes ~= vol;
328     }
329 }
330 
331 void mergeVolumeInfoAndStatus(ref Volume[] volumes, Brick[string] bricksdata)
332 {
333     foreach(vol; volumes)
334     {
335         foreach(subvol; vol.subvols)
336         {
337             foreach(ref brick; subvol.bricks)
338             {
339                 auto name = brick.host ~ ":" ~ brick.path;
340                 brick.volumeId = vol.id;
341                 brick.subvolId = subvol.id;
342                 if ((name in bricksdata) !is null)
343                 {
344                     brick.fsType = bricksdata[name].fsType;
345                     brick.device = bricksdata[name].device;
346                     brick.blockSize = bricksdata[name].blockSize;
347                     brick.mountOptions = bricksdata[name].mountOptions;
348                     brick.state = bricksdata[name].state;
349                     brick.sizeUsed = bricksdata[name].sizeUsed;
350                     brick.sizeTotal = bricksdata[name].sizeTotal;
351                     brick.inodesUsed = bricksdata[name].inodesUsed;
352                     brick.inodesTotal = bricksdata[name].inodesTotal;
353                     brick.port = bricksdata[name].port;
354                     brick.pid = bricksdata[name].pid;
355                 }
356             }
357         }
358     }
359 }
360 
361 void updateVolumeUtilization(ref Volume[] volumes)
362 {
363     foreach(ref vol; volumes)
364     {
365         foreach(subvol; vol.subvols)
366         {
367             ulong effectiveCapacityUsed, effectiveCapacityTotal;
368             ulong effectiveInodesUsed, effectiveInodesTotal;
369 
370             foreach(brick; subvol.bricks)
371             {
372                 if (brick.type != BrickTypeArbiter)
373                 {
374                     if(brick.sizeUsed >= effectiveCapacityUsed)
375                         effectiveCapacityUsed = brick.sizeUsed;
376                 
377                     if (effectiveCapacityTotal == 0 || brick.sizeTotal <= effectiveCapacityTotal)
378                         effectiveCapacityTotal = brick.sizeTotal;
379 
380                     if(brick.inodesUsed >= effectiveInodesUsed)
381                         effectiveInodesUsed = brick.inodesUsed;
382                 
383                     if (effectiveInodesTotal == 0 || brick.inodesTotal <= effectiveInodesTotal)
384                         effectiveInodesTotal = brick.inodesTotal;
385                 }
386             }
387             if (subvol.type == SubvolTypeDisp)
388             {
389                 // Subvol Size = Sum of size of Data bricks
390                 effectiveCapacityUsed = effectiveCapacityUsed * (subvol.disperseCount - subvol.disperseRedundancyCount);
391                 effectiveCapacityTotal = effectiveCapacityTotal * (subvol.disperseCount - subvol.disperseRedundancyCount);
392                 effectiveInodesUsed = effectiveInodesUsed * (subvol.disperseCount - subvol.disperseRedundancyCount);
393                 effectiveInodesTotal = effectiveInodesTotal * (subvol.disperseCount - subvol.disperseRedundancyCount);
394             }
395 
396             vol.sizeTotal += effectiveCapacityTotal;
397             vol.sizeUsed += effectiveCapacityUsed;
398             vol.inodesTotal += effectiveInodesTotal;
399             vol.inodesUsed += effectiveInodesUsed;
400         }
401     }
402 }
403 
404 void updateVolumeHealth(ref Volume[] volumes)
405 {
406     foreach(ref vol; volumes)
407     {
408         if (vol.state != StateStarted)
409             continue;
410 
411         vol.health = HealthUp;
412         auto upSubvols = 0;
413         foreach(ref subvol; vol.subvols)
414         {
415             auto upBricks = 0;
416             foreach(brick; subvol.bricks)
417             {
418                 if (brick.state == HealthUp)
419                     upBricks++;
420             }
421             subvol.health = HealthUp;
422             if (subvol.bricks.length != upBricks)
423             {
424                 subvol.health = HealthDown;
425                 if (subvol.type == SubvolTypeRep && upBricks >= (subvol.replicaCount/2 + 1))
426                     subvol.health = HealthPartial;
427 
428                 // If down bricks are less than or equal to redudancy count
429                 // then Volume is UP but some bricks are down
430                 if (subvol.type == SubvolTypeDisp &&
431                     (subvol.bricks.length - upBricks) <= subvol.disperseRedundancyCount)
432                     subvol.health = HealthPartial;
433 
434             }
435 
436             if (subvol.health == HealthDown)
437                 vol.health = HealthDegraded;
438 
439             if (subvol.health == HealthPartial && vol.health != HealthDegraded)
440                 vol.health = subvol.health;
441 
442             if (subvol.health != HealthDown)
443                 upSubvols++;
444         }
445         if (upSubvols == 0)
446             vol.health = HealthDown;
447     }
448 }
449 
450 
451 void parseVolumesFromStatus(string[] output, ref Volume[] volumes)
452 {
453     auto document = new Document(output.join(""));
454     auto bricksdata = parseBricksFromVolStatus(document.getElementsByTagName("node"));
455     mergeVolumeInfoAndStatus(volumes, bricksdata);
456     updateVolumeUtilization(volumes);
457     updateVolumeHealth(volumes);
458 }