Skip to content

Products / LILOM

Go to a project
Toggle navigation
Toggle navigation pinning
Commit d873e5f5 d873e5f5ca1c4cffeb1ab0ad005bac1db000c50f by Christian Gerdes
Options

Ny version som hämtar GC info 

Dock räknar den fel på Allocation Rate när heap size minskar från
föregående sampling.
1 parent f95dafd9
Inline Side-by-side
Showing 4 changed files with 268 additions and 31 deletions
1 javaw.exe -Dcom.sun.management.jmxremote.ssl=false -cp "wls-10.3.6.0.161018.2/wljmxclient.jar;jconsole.1.8.0_91.jar" sun.tools.jconsole.JConsole "service:jmx:rmi:///jndi/iiop://u01891.ef.kap.rsv.se:17020/weblogic.management.mbeanservers.runtime"
...\ No newline at end of file ...\ No newline at end of file
1 javaw.exe -Dcom.sun.management.jmxremote.ssl=false -cp "wls-10.3.6.0.161018.2/wljmxclient.jar;jconsole.1.8.0_91.jar" sun.tools.jconsole.JConsole "service:jmx:rmi:///jndi/iiop://u02878.ht.kap.rsv.se:17040/weblogic.management.mbeanservers.runtime"
...\ No newline at end of file ...\ No newline at end of file
...@@ -3,15 +3,12 @@ package se.lil.jm; ...@@ -3,15 +3,12 @@ package se.lil.jm;
3 import java.io.IOException; 3 import java.io.IOException;
4 import java.net.MalformedURLException; 4 import java.net.MalformedURLException;
5 import java.util.HashMap; 5 import java.util.HashMap;
6 import java.util.Set; 6 import java.util.HashSet;
7 import java.util.concurrent.locks.Lock; 7 import java.util.concurrent.locks.Lock;
8 import java.util.concurrent.locks.ReentrantLock; 8 import java.util.concurrent.locks.ReentrantLock;
9 9
10 import javax.management.JMX; 10 import javax.management.InstanceNotFoundException;
11 import javax.management.MBeanAttributeInfo;
12 import javax.management.MBeanInfo;
13 import javax.management.MBeanServerConnection; 11 import javax.management.MBeanServerConnection;
14 import javax.management.MalformedObjectNameException;
15 import javax.management.ObjectName; 12 import javax.management.ObjectName;
16 import javax.management.remote.JMXConnector; 13 import javax.management.remote.JMXConnector;
17 import javax.management.remote.JMXConnectorFactory; 14 import javax.management.remote.JMXConnectorFactory;
...@@ -26,6 +23,19 @@ public class JmxMon { ...@@ -26,6 +23,19 @@ public class JmxMon {
26 String serverName = null; 23 String serverName = null;
27 String runtimeName = null; 24 String runtimeName = null;
28 25
26 int gcType = 0;
27 public static final int Unknown = 0;
28 public static final int CMS = 1;
29 public static final int PS = 2;
30
31 public boolean isGCType(int type) {
32 return (gcType == type);
33 }
34
35 private void setGCType(int type) {
36 gcType = type;
37 }
38
29 JMXServiceURL url = null; 39 JMXServiceURL url = null;
30 JMXConnector jmxc = null; 40 JMXConnector jmxc = null;
31 MBeanServerConnection mbsc = null; 41 MBeanServerConnection mbsc = null;
...@@ -39,6 +49,19 @@ public class JmxMon { ...@@ -39,6 +49,19 @@ public class JmxMon {
39 HashMap<String, LongDelta> longDeltaMap = new HashMap<String, LongDelta>(); 49 HashMap<String, LongDelta> longDeltaMap = new HashMap<String, LongDelta>();
40 HashMap<String, DoubleDelta> doubleDeltaMap = new HashMap<String, DoubleDelta>(); 50 HashMap<String, DoubleDelta> doubleDeltaMap = new HashMap<String, DoubleDelta>();
41 51
52 HashSet<String> blocked = new HashSet<String>();
53
54 public void dumpDeltas() {
55 System.out.println("\nLong Deltas:");
56 for(String key : longDeltaMap.keySet()) {
57 System.out.println("\"" + key + "\" " + longDeltaMap.get(key).getCurrentValue());
58 }
59 System.out.println("\nDouble Deltas:");
60 for(String key : doubleDeltaMap.keySet()) {
61 System.out.println("\"" + key + "\" " + doubleDeltaMap.get(key).getCurrentValue());
62 }
63 }
64
42 private void updateLongDelta (Long timestamp, String name, Long value) { 65 private void updateLongDelta (Long timestamp, String name, Long value) {
43 LongDelta ld = longDeltaMap.get(name); 66 LongDelta ld = longDeltaMap.get(name);
44 if(ld == null) { 67 if(ld == null) {
...@@ -52,6 +75,10 @@ public class JmxMon { ...@@ -52,6 +75,10 @@ public class JmxMon {
52 return longDeltaMap.get(name + ":" + attribute); 75 return longDeltaMap.get(name + ":" + attribute);
53 } 76 }
54 77
78 public LongDelta getLongDelta(String key) {
79 return longDeltaMap.get(key);
80 }
81
55 private void updateDoubleDelta (Long timestamp, String name, Double value) { 82 private void updateDoubleDelta (Long timestamp, String name, Double value) {
56 DoubleDelta dd = doubleDeltaMap.get(name); 83 DoubleDelta dd = doubleDeltaMap.get(name);
57 if(dd == null) { 84 if(dd == null) {
...@@ -65,37 +92,78 @@ public class JmxMon { ...@@ -65,37 +92,78 @@ public class JmxMon {
65 return doubleDeltaMap.get(name + ":" + attribute); 92 return doubleDeltaMap.get(name + ":" + attribute);
66 } 93 }
67 94
95 public DoubleDelta getDoubleDelta(String key) {
96 return doubleDeltaMap.get(key);
97 }
98
68 private Object getAttributeValue(String name, String attribute) { 99 private Object getAttributeValue(String name, String attribute) {
69 Object ro = null; 100 Object ro = null;
70 if(mbsc != null) { 101 if(mbsc != null) {
71 try { 102 try {
72 ro = mbsc.getAttribute(new ObjectName(name), attribute); 103 if(!blocked.contains(name)) ro = mbsc.getAttribute(new ObjectName(name), attribute);
104 } catch (InstanceNotFoundException ie) {
105 blocked.add(name);
73 } catch (Exception e) { 106 } catch (Exception e) {
74 ro = e;
75 } 107 }
76 } 108 }
77 return ro; 109 return ro;
78 } 110 }
79 111
112 String lastKeyName = "";
113
114 private void updateCompositeDelta (Long timestamp, String name, CompositeData data) {
115 for(String key : data.getCompositeType().keySet()) {
116 String fullName;
117 if(key.equals("value")) fullName = name + ":" + lastKeyName;
118 else fullName = name + ":" + key;
119 Object obj = data.get(key);
120 updateDelta(timestamp, fullName, obj);
121 }
122 }
123
124 private void updateTabularDelta (Long timestamp, String name, TabularData data) {
125 for(Object obj : data.values()) {
126 CompositeData cd = (CompositeData)obj;
127 updateCompositeDelta(timestamp, name, cd);
128 }
129 }
130
131 private void updateDelta(Long timestamp, String fullName, Object data) {
132 if(data != null) {
133 if(data.getClass().equals(java.lang.String.class)) {
134 lastKeyName = (String)data;
135 } else
136 if(data.getClass().equals(java.lang.Long.class)) {
137 updateLongDelta(timestamp, fullName, (Long)data);
138 } else
139 if(data.getClass().equals(java.lang.Integer.class)) {
140 Integer i = (Integer)data;
141 updateLongDelta(timestamp, fullName, i.longValue());
142 } else
143 if(data.getClass().equals(java.lang.Double.class)) {
144 updateDoubleDelta(timestamp, fullName, (Double)data);
145 } else
146 if(data.getClass().equals(java.lang.Float.class)) {
147 Float f = (Float)data;
148 updateDoubleDelta(timestamp, fullName, f.doubleValue());
149 } else
150 if(data.getClass().equals(javax.management.openmbean.CompositeDataSupport.class)) {
151 updateCompositeDelta(timestamp, fullName, (CompositeData)data);
152 } else
153 if(data.getClass().equals(javax.management.openmbean.TabularDataSupport.class)) {
154 updateTabularDelta(timestamp, fullName, (TabularData)data);
155 } else {
156 System.out.println("Received Unknown Object: " + data.getClass());
157 }
158 }
159 }
160
80 private void fetchUpdateAttribute(String name, String attribute) { 161 private void fetchUpdateAttribute(String name, String attribute) {
81 String fullName = name + ":" + attribute; 162 String fullName = name + ":" + attribute;
82 Object obj = getAttributeValue(name, attribute); 163 Object obj = getAttributeValue(name, attribute);
83 Long ts = (Long)getAttributeValue("java.lang:type=Runtime", "Uptime");
84 if(obj != null) { 164 if(obj != null) {
85 if(obj.getClass().equals(java.lang.Long.class)) { 165 Long ts = (Long)getAttributeValue("java.lang:type=Runtime", "Uptime");
86 updateLongDelta(ts, fullName, (Long)obj); 166 updateDelta(ts, fullName, obj);
87 }
88 if(obj.getClass().equals(java.lang.Integer.class)) {
89 Integer i = (Integer)obj;
90 updateLongDelta(ts, fullName, i.longValue());
91 }
92 if(obj.getClass().equals(java.lang.Double.class)) {
93 updateDoubleDelta(ts, fullName, (Double)obj);
94 }
95 if(obj.getClass().equals(java.lang.Float.class)) {
96 Float f = (Float)obj;
97 updateDoubleDelta(ts, fullName, f.doubleValue());
98 }
99 } 167 }
100 } 168 }
101 169
...@@ -204,10 +272,31 @@ public class JmxMon { ...@@ -204,10 +272,31 @@ public class JmxMon {
204 fetchUpdateAttribute("java.lang:type=OperatingSystem", "SystemLoadAverage"); 272 fetchUpdateAttribute("java.lang:type=OperatingSystem", "SystemLoadAverage");
205 fetchUpdateAttribute("java.lang:type=Threading", "ThreadCount"); 273 fetchUpdateAttribute("java.lang:type=Threading", "ThreadCount");
206 fetchUpdateAttribute("java.lang:type=ClassLoading", "LoadedClassCount"); 274 fetchUpdateAttribute("java.lang:type=ClassLoading", "LoadedClassCount");
275 fetchUpdateAttribute("java.lang:type=Memory", "HeapMemoryUsage");
276
277 // Only get theese if we have the right GC type
278 if(isGCType(JmxMon.Unknown) || isGCType(JmxMon.PS)) {
279 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=PS MarkSweep", "LastGcInfo");
280 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=PS MarkSweep", "CollectionTime");
281 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=PS Scavenge", "CollectionTime");
282 fetchUpdateAttribute("java.lang:type=MemoryPool,name=PS Old Gen", "Usage");
283 }
284 if(isGCType(JmxMon.Unknown) || isGCType(JmxMon.CMS)) {
285 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", "LastGcInfo");
286 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", "CollectionTime");
287 fetchUpdateAttribute("java.lang:type=GarbageCollector,name=ParNew", "CollectionTime");
288 fetchUpdateAttribute("java.lang:type=MemoryPool,name=CMS Old Gen", "Usage");
289 }
207 // Finished updating data 290 // Finished updating data
208 getDataSucess++; 291 getDataSucess++;
209 lastFetchTSns = System.nanoTime(); 292 lastFetchTSns = System.nanoTime();
210 lastRTns = lastFetchTSns - startTSns; 293 lastRTns = lastFetchTSns - startTSns;
294
295 // Check the GC type if not known
296 if(isGCType(JmxMon.Unknown)) {
297 if(longDeltaMap.containsKey("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:CollectionTime")) setGCType(JmxMon.CMS);
298 if(longDeltaMap.containsKey("java.lang:type=GarbageCollector,name=PS MarkSweep:CollectionTime")) setGCType(JmxMon.PS);
299 }
211 return true; 300 return true;
212 } catch (Throwable e) { 301 } catch (Throwable e) {
213 throw (e); 302 throw (e);
...@@ -276,6 +365,132 @@ public class JmxMon { ...@@ -276,6 +365,132 @@ public class JmxMon {
276 return (open/max) * 100D; 365 return (open/max) * 100D;
277 } 366 }
278 367
368 public double getHeapUsedMB() {
369 LongDelta ld = getLongDelta("java.lang:type=Memory:HeapMemoryUsage:used");
370 if(ld != null) {
371 return (double)ld.getCurrentValue() / (1024D*1024D);
372 } else {
373 return 0;
374 }
375 }
376
377 public double getHeapUsedPercent() {
378 LongDelta ldu = getLongDelta("java.lang:type=Memory:HeapMemoryUsage:used");
379 LongDelta ldm = getLongDelta("java.lang:type=Memory:HeapMemoryUsage:max");
380 if(ldu != null && ldm != null) {
381 return ((double)ldu.getCurrentValue()/(double)ldm.getCurrentValue()) * 100D;
382 } else {
383 return 0;
384 }
385 }
386
387 public double getHeapAllocationRateMBps() {
388 LongDelta ldu = getLongDelta("java.lang:type=Memory:HeapMemoryUsage:used");
389 if(ldu != null) {
390 if(ldu.getPerSecondValue() > 0) {
391 return ldu.getPerSecondValue()/(1024D*1024D);
392 } else {
393 // negative delta, means we had a major GC in the delta, get the last heap size before GC and add it to the last (previous) value
394 Long beforeGC = null;
395 if(isGCType(JmxMon.CMS)) {
396 LongDelta ld1 = getLongDelta("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:LastGcInfo:memoryUsageBeforeGc:CMS Old Gen:used");
397 LongDelta ld2 = getLongDelta("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:LastGcInfo:memoryUsageBeforeGc:Par Eden Space:used");
398 LongDelta ld3 = getLongDelta("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:LastGcInfo:memoryUsageBeforeGc:Par Survivor Space:used");
399 if(ld1 != null && ld2 != null && ld3 != null)
400 beforeGC = ld1.getCurrentValue() + ld2.getCurrentValue() + ld3.getCurrentValue();
401 } else
402 if(isGCType(JmxMon.PS)) {
403 LongDelta ld1 = getLongDelta("java.lang:type=GarbageCollector,name=PS MarkSweep:LastGcInfo:memoryUsageBeforeGc:PS Old Gen:used");
404 LongDelta ld2 = getLongDelta("java.lang:type=GarbageCollector,name=PS MarkSweep:LastGcInfo:memoryUsageBeforeGc:PS Eden Space:used");
405 LongDelta ld3 = getLongDelta("java.lang:type=GarbageCollector,name=PS MarkSweep:LastGcInfo:memoryUsageBeforeGc:PS Survivor Space:used");
406 if(ld1 != null && ld2 != null && ld3 != null)
407 beforeGC = ld1.getCurrentValue() + ld2.getCurrentValue() + ld3.getCurrentValue();
408 }
409 if(beforeGC == null) return 0;
410 if(ldu.lastValue == null) return 0;
411 ldu.lastValue = beforeGC + ldu.lastValue;
412 System.out.println("Adjusted heap size with " + beforeGC + " bytes");
413 return ldu.getPerSecondValue()/(1024D*1024D);
414 }
415 } else {
416 return 0;
417 }
418 }
419
420 public double getPSOldGenUsedMB() {
421 LongDelta ld = getLongDelta("java.lang:type=MemoryPool,name=PS Old Gen:Usage:used");
422 if(ld != null) {
423 return (double)ld.getCurrentValue() / (1024D*1024D);
424 } else {
425 return 0;
426 }
427 }
428
429 public double getCMSOldGenUsedMB() {
430 LongDelta ld = getLongDelta("java.lang:type=MemoryPool,name=CMS Old Gen:Usage:used");
431 if(ld != null) {
432 return (double)ld.getCurrentValue() / (1024D*1024D);
433 } else {
434 return 0;
435 }
436 }
437
438 public double getPSOldGenUsedPercent() {
439 LongDelta ldu = getLongDelta("java.lang:type=MemoryPool,name=PS Old Gen:Usage:used");
440 LongDelta ldm = getLongDelta("java.lang:type=MemoryPool,name=PS Old Gen:Usage:max");
441 if(ldu != null && ldm != null) {
442 return ((double)ldu.getCurrentValue()/(double)ldm.getCurrentValue()) * 100D;
443 } else {
444 return 0;
445 }
446 }
447
448 public double getCMSOldGenUsedPercent() {
449 LongDelta ldu = getLongDelta("java.lang:type=MemoryPool,name=CMS Old Gen:Usage:used");
450 LongDelta ldm = getLongDelta("java.lang:type=MemoryPool,name=CMS Old Gen:Usage:max");
451 if(ldu != null && ldm != null) {
452 return ((double)ldu.getCurrentValue()/(double)ldm.getCurrentValue()) * 100D;
453 } else {
454 return 0;
455 }
456 }
457
458 public double getPSMarkSweepOldGenAfterGCMB() {
459 LongDelta ld = getLongDelta("java.lang:type=GarbageCollector,name=PS MarkSweep:LastGcInfo:memoryUsageAfterGc:PS Old Gen:used");
460 if(ld != null) {
461 return (double)ld.getCurrentValue() / (1024D*1024D);
462 } else {
463 return 0;
464 }
465 }
466
467 public double getCMSMarkSweepOldGenAfterGCMB() {
468 LongDelta ld = getLongDelta("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:LastGcInfo:memoryUsageAfterGc:CMS Old Gen:used");
469 if(ld != null) {
470 return (double)ld.getCurrentValue() / (1024D*1024D);
471 } else {
472 return 0;
473 }
474 }
475
476 public double getPSMarkSweepPermGenAfterGCMB() {
477 LongDelta ld = getLongDelta("java.lang:type=GarbageCollector,name=PS MarkSweep:LastGcInfo:memoryUsageAfterGc:PS Perm Gen:used");
478 if(ld != null) {
479 return (double)ld.getCurrentValue() / (1024D*1024D);
480 } else {
481 return 0;
482 }
483 }
484
485 public double getCMSMarkSweepPermGenAfterGCMB() {
486 LongDelta ld = getLongDelta("java.lang:type=GarbageCollector,name=ConcurrentMarkSweep:LastGcInfo:memoryUsageAfterGc:CMS Perm Gen:used");
487 if(ld != null) {
488 return (double)ld.getCurrentValue() / (1024D*1024D);
489 } else {
490 return 0;
491 }
492 }
493
279 public JmxMon() { 494 public JmxMon() {
280 } 495 }
281 496
......
...@@ -19,9 +19,12 @@ public class TestRunner { ...@@ -19,9 +19,12 @@ public class TestRunner {
19 19
20 Long ts1 = System.currentTimeMillis(); 20 Long ts1 = System.currentTimeMillis();
21 21
22 JmxMon mon1 = new JmxMon("service:jmx:iiop:///jndi/iiop://u30457:29722/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2 22 //JmxMon mon1 = new JmxMon("service:jmx:iiop:///jndi/iiop://u30457:29722/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2
23 //JmxMon mon1 = new JmxMon("service:jmx:rmi:///jndi/iiop://u30128:23032/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2 23 //JmxMon mon1 = new JmxMon("service:jmx:rmi:///jndi/iiop://u30128:23032/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2
24 //JmxMon mon2 = new JmxMon("service:jmx:rmi:///jndi/iiop://u30009:26732/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.12.1 24 //JmxMon mon2 = new JmxMon("service:jmx:rmi:///jndi/iiop://u30009:26732/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.12.1
25 //JmxMon mon1 = new JmxMon("service:jmx:rmi:///jndi/iiop://u02878.ht.kap.rsv.se:17040/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2 PS
26 //JmxMon mon1 = new JmxMon("service:jmx:rmi:///jndi/iiop://u01891.ef.kap.rsv.se:17020/weblogic.management.mbeanservers.runtime"); // 10.3.6.0.161018.2 CMS
27 JmxMon mon1 = new JmxMon("service:jmx:rmi:///jndi/iiop://u30450:34502/weblogic.management.mbeanservers.runtime"); // CMS
25 mon1.open(); 28 mon1.open();
26 //mon2.open(); 29 //mon2.open();
27 jmxList.add(mon1); 30 jmxList.add(mon1);
...@@ -41,16 +44,33 @@ public class TestRunner { ...@@ -41,16 +44,33 @@ public class TestRunner {
41 mon.getData(); 44 mon.getData();
42 time = System.currentTimeMillis() - ts1; 45 time = System.currentTimeMillis() - ts1;
43 System.out.println(mon.getServerName() + " called in " + time + "ms"); 46 System.out.println(mon.getServerName() + " called in " + time + "ms");
44 System.out.println(mon.getServerName() + " Process CPU %: " + mon.getCPUPercent()); 47 System.out.println(" Process CPU %: " + mon.getCPUPercent());
45 System.out.println(mon.getServerName() + " Process Threads #: " + mon.getThreads()); 48 System.out.println(" Process Threads #: " + mon.getThreads());
46 System.out.println(mon.getServerName() + " Process Loaded Classes #: " + mon.getLoadedClassCount()); 49 System.out.println(" Process Loaded Classes #: " + mon.getLoadedClassCount());
47 System.out.println(mon.getServerName() + " Process File Descriptors %: " + mon.getSystemFileDescriptorsPercentUsed()); 50 System.out.println(" Process File Descriptors %: " + mon.getSystemFileDescriptorsPercentUsed());
48 System.out.println(mon.getServerName() + " System Load (per cpu): " + mon.getAverageLoadPerCpu()); 51 System.out.println(" Process Heap Used %: " + mon.getHeapUsedPercent());
49 System.out.println(mon.getServerName() + " System Load (total): " + mon.getAverageLoad()); 52 System.out.println(" Process Heap Used MB: " + mon.getHeapUsedMB());
50 System.out.println(mon.getServerName() + " System Memory %: " + mon.getSystemMemoryPercentUsed()); 53 System.out.println(" Allocation Rate MB/s: " + mon.getHeapAllocationRateMBps());
51 System.out.println(mon.getServerName() + " System Swap %: " + mon.getSystemSwapPercentUsed()); 54 if(mon.isGCType(JmxMon.PS)) {
55 System.out.println(" PS OldGen Used %: " + mon.getPSOldGenUsedPercent());
56 System.out.println(" PS OldGen Used MB: " + mon.getPSOldGenUsedMB());
57 System.out.println(" PS OldGen After Major GC MB: " + mon.getPSMarkSweepOldGenAfterGCMB());
58 System.out.println(" PS PermGen After Major GC MB: " + mon.getPSMarkSweepPermGenAfterGCMB());
59 }
60 if(mon.isGCType(JmxMon.CMS)) {
61 System.out.println(" CMS OldGen Used %: " + mon.getCMSOldGenUsedPercent());
62 System.out.println(" CMS OldGen Used MB: " + mon.getCMSOldGenUsedMB());
63 System.out.println(" CMS OldGen After Major GC MB: " + mon.getCMSMarkSweepOldGenAfterGCMB());
64 System.out.println("CMS PermGen After Major GC MB: " + mon.getCMSMarkSweepPermGenAfterGCMB());
65 }
66 System.out.println(" System Load (per cpu): " + mon.getAverageLoadPerCpu());
67 System.out.println(" System Load (total): " + mon.getAverageLoad());
68 System.out.println(" System Memory %: " + mon.getSystemMemoryPercentUsed());
69 System.out.println(" System Swap %: " + mon.getSystemSwapPercentUsed());
52 70
53 System.out.println(); 71 System.out.println();
72
73 //mon.dumpDeltas();
54 } 74 }
55 Thread.sleep(15000); 75 Thread.sleep(15000);
56 } 76 }
......