Skip to content

Products / LILOM

Go to a project
Toggle navigation
Toggle navigation pinning
Commit ab86ea2f ab86ea2f66fa71520c736d129d3b74a6ecf092b0 by Christian Gerdes
Options

First version with support for the "Oracle Bug"

1 parent a578de84
Inline Side-by-side
Showing 6 changed files with 137 additions and 109 deletions
...@@ -66,3 +66,13 @@ inte i alla metoder i OraMon.java klassen). Anger man en PDB i OraMon.jsp så bl ...@@ -66,3 +66,13 @@ inte i alla metoder i OraMon.java klassen). Anger man en PDB i OraMon.jsp så bl
66 Lyckades köra JavaMonWeb i Debug läge med Tomcat 7 local server. Skapade en sanityCheck() metod med break points så jag kan debugga när vissa räknare blir negativa. 66 Lyckades köra JavaMonWeb i Debug läge med Tomcat 7 local server. Skapade en sanityCheck() metod med break points så jag kan debugga när vissa räknare blir negativa.
67 Dock svårt att hitta rätt LongDelta objekt, eftersom det finns över 1000 i listan i random ordning. Behöver spara undan det nånstans eller breaka inne i en LongDelta metod istället. 67 Dock svårt att hitta rätt LongDelta objekt, eftersom det finns över 1000 i listan i random ordning. Behöver spara undan det nånstans eller breaka inne i en LongDelta metod istället.
68 Checkar in detta i alla fall så länge. 68 Checkar in detta i alla fall så länge.
69
70 2019-01-23
71
72 Oracle support var ingen väg framåt. Planen är att bygga en work around.
73 Men det blir inte så lätt. Koden just nu har "underrun protection" påslaget default, men det fungerar inte eftersom flera räknare "inte" är cumulativa utan ska kunna ha minskande värden.
74 Därmed blir det svårt att skydda sig mot detta under en update.
75 Istället måste detta detekteras när man läser av ett delta i någon av delta metoderna, varvid det dock är för sent.
76 Innan trodde jag att problemet alltid inleds med ett negativt delta, sedan en spik, sedan normalt. Men så är inte fallet, jag har sett idag tvärtom, det inleds med en spik och sedan
77 en rättning (vilket känns konstigt) men så blev det på tex file io wait time. Så jag tror tyvärr att enda rätta lösningen på problemet är att spara minst 3 mätningar innan ett delta rapporteras,
78 samt att man då kan upptäcka ett negativt delta, och i så fall ignorera värdet innan.
......
1 package se.lil.om; 1 package se.lil.om;
2 2
3 import java.io.BufferedWriter;
4 import java.io.FileWriter;
3 import java.math.BigDecimal; 5 import java.math.BigDecimal;
4 import java.sql.ResultSet; 6 import java.sql.ResultSet;
5 import java.sql.Timestamp; 7 import java.sql.Timestamp;
8 import java.text.SimpleDateFormat;
6 import java.util.ArrayList; 9 import java.util.ArrayList;
10 import java.util.Calendar;
7 11
8 class Collector { 12 class Collector {
9 ArrayList<LongDelta> list = new ArrayList<LongDelta>(); 13 ArrayList<LongDelta> list = new ArrayList<LongDelta>();
14 String name = "";
15 Boolean debugLog = false;
16
17 public Collector(String name) {
18 this.name = name;
19 }
10 20
11 public void update(ResultSet rset) throws Throwable { 21 public void update(ResultSet rset) throws Throwable {
22 String timeStamp = null;
23 BufferedWriter writer = null;
24
25 if(debugLog) {
26 timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(Calendar.getInstance().getTime());
27 writer = new BufferedWriter(new FileWriter("Collector_" + name + "_" + timeStamp + ".txt"));
28 }
12 while(rset.next()) { 29 while(rset.next()) {
13 // Add try catch for issue #5 30 // Add try catch for issue #5
14 Timestamp ts = rset.getTimestamp(1); 31 Timestamp ts = rset.getTimestamp(1);
...@@ -21,27 +38,29 @@ class Collector { ...@@ -21,27 +38,29 @@ class Collector {
21 value = BigDecimal.ZERO; 38 value = BigDecimal.ZERO;
22 } 39 }
23 updateValue(ts, name, value); 40 updateValue(ts, name, value);
41 if(writer != null) {
42 writer.write(ts + ":" + name + ":" + value);
43 writer.newLine();
44 }
45 }
46 if(writer != null) {
47 writer.close();
24 } 48 }
25 rset.close(); 49 rset.close();
26 } 50 }
27 51
28 public void updateValue(Timestamp ts, String name, BigDecimal value) { 52 public void updateValue(Timestamp timestamp, String name, BigDecimal value) {
29 LongDelta myDelta = null; 53 LongDelta myDelta = null;
30 for(int x = 0; x<list.size() && myDelta == null; x++) { 54 for(int x = 0; x<list.size() && myDelta == null; x++) {
31 if(name.equals(list.get(x).name)) { 55 if(list.get(x).matches(name)) {
32 myDelta = list.get(x); 56 myDelta = list.get(x);
33 } 57 }
34 } 58 }
35 if(myDelta != null) { 59 if(myDelta != null) {
36 myDelta.lastFetch = myDelta.curFetch; 60 myDelta.update(value, timestamp);
37 myDelta.curFetch = ts;
38 myDelta.lastValue = myDelta.curValue;
39 myDelta.curValue = value;
40 } else { 61 } else {
41 myDelta = new LongDelta(); 62 myDelta = new LongDelta(name);
42 myDelta.curFetch = ts; 63 myDelta.update(value, timestamp);
43 myDelta.curValue = value;
44 myDelta.name = name;
45 list.add(myDelta); 64 list.add(myDelta);
46 } 65 }
47 } 66 }
...@@ -49,7 +68,7 @@ class Collector { ...@@ -49,7 +68,7 @@ class Collector {
49 public long getCurrentValue(String name) throws Throwable { 68 public long getCurrentValue(String name) throws Throwable {
50 LongDelta myDelta = null; 69 LongDelta myDelta = null;
51 for(int x = 0; x<list.size() && myDelta == null; x++) { 70 for(int x = 0; x<list.size() && myDelta == null; x++) {
52 if(name.equals(list.get(x).name)) { 71 if(list.get(x).matches(name)) {
53 myDelta = list.get(x); 72 myDelta = list.get(x);
54 } 73 }
55 } 74 }
...@@ -59,23 +78,23 @@ class Collector { ...@@ -59,23 +78,23 @@ class Collector {
59 return 0; 78 return 0;
60 } 79 }
61 80
62 public long getDelta(String name) throws Throwable { 81 // public long getDelta(String name) throws Throwable {
63 LongDelta myDelta = null; 82 // LongDelta myDelta = null;
64 for(int x = 0; x<list.size() && myDelta == null; x++) { 83 // for(int x = 0; x<list.size() && myDelta == null; x++) {
65 if(name.equals(list.get(x).name)) { 84 // if(list.get(x).matches(name)) {
66 myDelta = list.get(x); 85 // myDelta = list.get(x);
67 } 86 // }
68 } 87 // }
69 if(myDelta != null) 88 // if(myDelta != null)
70 return myDelta.getDelta(); 89 // return myDelta.getDelta();
71 else 90 // else
72 return 0; 91 // return 0;
73 } 92 // }
74 93
75 public double getPerSecValue(String name) throws Throwable { 94 public double getPerSecValue(String name) throws Throwable {
76 LongDelta myDelta = null; 95 LongDelta myDelta = null;
77 for(int x = 0; x<list.size() && myDelta == null; x++) { 96 for(int x = 0; x<list.size() && myDelta == null; x++) {
78 if(name.equals(list.get(x).name)) { 97 if(list.get(x).matches(name)) {
79 myDelta = list.get(x); 98 myDelta = list.get(x);
80 } 99 }
81 } 100 }
...@@ -88,7 +107,7 @@ class Collector { ...@@ -88,7 +107,7 @@ class Collector {
88 public double getSeconds(String name) throws Throwable { 107 public double getSeconds(String name) throws Throwable {
89 LongDelta myDelta = null; 108 LongDelta myDelta = null;
90 for(int x = 0; x<list.size() && myDelta == null; x++) { 109 for(int x = 0; x<list.size() && myDelta == null; x++) {
91 if(name.equals(list.get(x).name)) { 110 if(list.get(x).matches(name)) {
92 myDelta = list.get(x); 111 myDelta = list.get(x);
93 } 112 }
94 } 113 }
......
...@@ -5,17 +5,25 @@ import java.sql.ResultSet; ...@@ -5,17 +5,25 @@ import java.sql.ResultSet;
5 import java.sql.Timestamp; 5 import java.sql.Timestamp;
6 6
7 class LongDelta { 7 class LongDelta {
8 public String name = null; 8 private String name = null;
9 public BigDecimal curValue = null; 9
10 public BigDecimal lastValue = null; 10 private BigDecimal curValue = null;
11 public Timestamp lastFetch = null; 11 private BigDecimal lastValue = null;
12 public Timestamp curFetch = null; 12 private BigDecimal last2Value = null;
13 boolean urProt = false; // Under-run protection 13
14 boolean lastUpdCausedUR = false; 14 private Timestamp curFetch = null;
15 public static BigDecimal bd1000 = new BigDecimal(1000); 15 private Timestamp lastFetch = null;
16 16 private Timestamp last2Fetch = null;
17 public LongDelta() {}; 17
18 public LongDelta(boolean UnderrunProtection) { this.urProt = UnderrunProtection; } 18 private static BigDecimal bd1000 = new BigDecimal(1000);
19
20 private double numMilliSeconds;
21 private double delta;
22 private boolean haveGoodDelta = false;
23
24 public LongDelta(String name) {
25 this.name = name;
26 }
19 27
20 public void update(ResultSet rset) throws Throwable{ 28 public void update(ResultSet rset) throws Throwable{
21 update(rset, 2, false, true); 29 update(rset, 2, false, true);
...@@ -32,63 +40,80 @@ class LongDelta { ...@@ -32,63 +40,80 @@ class LongDelta {
32 if(close) rset.close(); 40 if(close) rset.close();
33 } 41 }
34 42
43 public boolean matches(String name) {
44 if(this.name.equals(name)) return true;
45 else return false;
46 }
47
35 public void update(BigDecimal value, Timestamp timestamp) { 48 public void update(BigDecimal value, Timestamp timestamp) {
36 update(value, timestamp, false); 49 update(value, timestamp, false);
37 } 50 }
38 51
39 public void update(BigDecimal value, Timestamp timestamp, boolean convert) { 52 public void update(BigDecimal value, Timestamp timestamp, boolean convert) {
40 if(convert) value = value.divide(bd1000, BigDecimal.ROUND_HALF_UP); 53 if(convert) value = value.divide(bd1000, BigDecimal.ROUND_HALF_UP);
41 if(urProt) { 54 this.last2Fetch = this.lastFetch;
42 if(this.curValue != null && this.curValue.compareTo(value) > 0) {
43 //This would be a negative delta. Ignore this data
44 //System.out.println("Underrunprotection on " + name + " New Val: " + newval + " Old Val: " + this.curValue);
45 lastUpdCausedUR = true;
46 return;
47 }
48 }
49 this.lastFetch = this.curFetch; 55 this.lastFetch = this.curFetch;
50 this.curFetch = timestamp; 56 this.curFetch = timestamp;
57 this.last2Value = this.lastValue;
51 this.lastValue = this.curValue; 58 this.lastValue = this.curValue;
52 this.curValue = value; 59 this.curValue = value;
53 lastUpdCausedUR = false; 60 this.haveGoodDelta = false;
54 } 61 }
55 62
56 public boolean didLastUnderRun() { 63 private void updateDelta() {
57 return lastUpdCausedUR; 64 if(this.curValue != null && this.lastValue != null && this.last2Value != null
65 && this.lastFetch != null && this.last2Fetch != null && this.curFetch != null) {
66
67 // We have values, calculate the number of gets per second
68
69 // Handle the Oracle Bug (ref needed)
70 if(curValue.compareTo(lastValue) >= 0 && lastValue.compareTo(last2Value) >= 0) {
71 // All seems fine, no negative deltas, return the delta for last and last2
72 numMilliSeconds = this.lastFetch.getTime() - this.last2Fetch.getTime();
73 delta = this.lastValue.subtract(this.last2Value).doubleValue();
74 } else {
75 // negative delta between current and last, or last and last2, return current and last2 instead
76 numMilliSeconds = this.curFetch.getTime() - this.last2Fetch.getTime();
77 delta = this.curValue.subtract(this.last2Value).doubleValue();
78 }
79
80 haveGoodDelta = true;
81 }
58 } 82 }
59 83
60 public double getPerSecondValue() throws Throwable { 84 public double getPerSecondValue() throws Throwable {
61 if(this.curValue != null && this.lastValue != null && this.lastFetch != null && this.curFetch != null) { 85 if(haveGoodDelta == false) updateDelta();
62 // We have values, calculate the number of gets per second 86 if(haveGoodDelta) {
63 double numMilliSeconds = this.curFetch.getTime() - this.lastFetch.getTime();
64 double delta = this.curValue.subtract(this.lastValue).doubleValue();
65 double perMilli = delta/numMilliSeconds; 87 double perMilli = delta/numMilliSeconds;
66 return perMilli * 1000; 88 return perMilli * 1000;
67 } else { 89 } else {
68 return 0; 90 return 0;
69 } 91 }
70 } 92 }
93
71 public long getCurrentValue() throws Throwable { 94 public long getCurrentValue() throws Throwable {
72 if(this.curValue != null) return this.curValue.longValue(); 95 if(this.curValue != null) return this.curValue.longValue();
73 else return 0; 96 else return 0;
74 } 97 }
98
75 public double getSeconds() throws Throwable { 99 public double getSeconds() throws Throwable {
76 if(this.lastFetch != null && this.curFetch != null) { 100 if(haveGoodDelta == false) updateDelta();
77 double numSeconds = this.curFetch.getTime() - this.lastFetch.getTime(); 101 if(haveGoodDelta) {
78 return numSeconds/1000; 102 return numMilliSeconds/1000;
79 } 103 } else {
80 return 0;
81 }
82 public long getMilliSeconds() throws Throwable {
83 if(this.lastFetch != null && this.curFetch != null) {
84 return this.curFetch.getTime() - this.lastFetch.getTime();
85 }
86 return 0; 104 return 0;
87 } 105 }
88 public long getDelta() throws Throwable {
89 if(this.lastValue != null && this.curValue != null) {
90 return this.curValue.subtract(this.lastValue).longValue();
91 }
92 return 0;
93 } 106 }
107 // public long getMilliSeconds() throws Throwable {
108 // if(this.lastFetch != null && this.curFetch != null) {
109 // return this.curFetch.getTime() - this.lastFetch.getTime();
110 // }
111 // return 0;
112 // }
113 // public long getDelta() throws Throwable {
114 // if(this.lastValue != null && this.curValue != null) {
115 // return this.curValue.subtract(this.lastValue).longValue();
116 // }
117 // return 0;
118 // }
94 } 119 }
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -5,12 +5,12 @@ import java.util.HashSet; ...@@ -5,12 +5,12 @@ import java.util.HashSet;
5 import java.util.Properties; 5 import java.util.Properties;
6 import java.util.concurrent.locks.Lock; 6 import java.util.concurrent.locks.Lock;
7 import java.util.concurrent.locks.ReentrantLock; 7 import java.util.concurrent.locks.ReentrantLock;
8 import java.util.logging.Logger; 8 //import java.util.logging.Logger;
9 import java.util.regex.Matcher; 9 import java.util.regex.Matcher;
10 import java.util.regex.Pattern; 10 import java.util.regex.Pattern;
11 11
12 public class OraMon { 12 public class OraMon {
13 private final static Logger LOGGER = Logger.getLogger(OraMon.class.getName()); 13 //private final static Logger LOGGER = Logger.getLogger(OraMon.class.getName());
14 Connection conn = null; 14 Connection conn = null;
15 String conString = "jdbc:oracle:thin:@//hostname:1521/SID"; 15 String conString = "jdbc:oracle:thin:@//hostname:1521/SID";
16 String conUser = "system"; 16 String conUser = "system";
...@@ -21,8 +21,8 @@ public class OraMon { ...@@ -21,8 +21,8 @@ public class OraMon {
21 public static final int SYSSTAT = 1; 21 public static final int SYSSTAT = 1;
22 public static final int OSSTAT = 2; 22 public static final int OSSTAT = 2;
23 23
24 Collector colSysStat = new Collector(); 24 Collector colSysStat = new Collector("colSysStat");
25 Collector colOsStat = new Collector(); 25 Collector colOsStat = new Collector("colOsStat");
26 26
27 int getDataCalls = 0; 27 int getDataCalls = 0;
28 int getDataSucess = 0; 28 int getDataSucess = 0;
...@@ -95,8 +95,8 @@ public class OraMon { ...@@ -95,8 +95,8 @@ public class OraMon {
95 } 95 }
96 96
97 //DB CPU 97 //DB CPU
98 LongDelta cpuTime = new LongDelta(true); 98 LongDelta cpuTime = new LongDelta(this.getFriendlyName()+" cpuTime");
99 LongDelta niwTime = new LongDelta(true); 99 LongDelta niwTime = new LongDelta(this.getFriendlyName()+" niwTime");
100 public double getCPUPercent(boolean excludeNonIdleWaitTime, String pdbName) throws Throwable { 100 public double getCPUPercent(boolean excludeNonIdleWaitTime, String pdbName) throws Throwable {
101 double cpuPerSec; 101 double cpuPerSec;
102 if(pdbName == null) cpuPerSec = cpuTime.getPerSecondValue(); 102 if(pdbName == null) cpuPerSec = cpuTime.getPerSecondValue();
...@@ -146,15 +146,15 @@ public class OraMon { ...@@ -146,15 +146,15 @@ public class OraMon {
146 return getCPURawValue(); 146 return getCPURawValue();
147 } 147 }
148 148
149 public long getCPURawDiff() throws Throwable { 149 // public long getCPURawDiff() throws Throwable {
150 return cpuTime.getDelta(); 150 // return cpuTime.getDelta();
151 } 151 // }
152 public long getCPURawDiff(String pdbName) throws Throwable { 152 // public long getCPURawDiff(String pdbName) throws Throwable {
153 if(pdbName != null) 153 // if(pdbName != null)
154 return colPdbCPU.getDelta(pdbName + strCpuTime); 154 // return colPdbCPU.getDelta(pdbName + strCpuTime);
155 else 155 // else
156 return getCPURawDiff(); 156 // return getCPURawDiff();
157 } 157 // }
158 158
159 public double getCPUTimePerSecond() throws Throwable { 159 public double getCPUTimePerSecond() throws Throwable {
160 return cpuTime.getPerSecondValue(); 160 return cpuTime.getPerSecondValue();
...@@ -408,7 +408,7 @@ public class OraMon { ...@@ -408,7 +408,7 @@ public class OraMon {
408 //Get PDB specific data if getPDB is set to true 408 //Get PDB specific data if getPDB is set to true
409 if(getPdbs) { 409 if(getPdbs) {
410 //CPU 410 //CPU
411 if(colPdbCPU == null) colPdbCPU = new Collector(); 411 if(colPdbCPU == null) colPdbCPU = new Collector("colPdbCPU");
412 rset = stmt.executeQuery("select systimestamp, c.NAME, s.value as NIWT, t.value as DBCPU from V$CON_SYSSTAT s, V$CON_SYS_TIME_MODEL t, V$CONTAINERS c where s.CON_ID=t.CON_ID and s.CON_ID=c.CON_ID and s.name='non-idle wait time' and t.STAT_NAME='DB CPU'"); 412 rset = stmt.executeQuery("select systimestamp, c.NAME, s.value as NIWT, t.value as DBCPU from V$CON_SYSSTAT s, V$CON_SYS_TIME_MODEL t, V$CONTAINERS c where s.CON_ID=t.CON_ID and s.CON_ID=c.CON_ID and s.name='non-idle wait time' and t.STAT_NAME='DB CPU'");
413 while(rset.next()) { 413 while(rset.next()) {
414 Timestamp ts = rset.getTimestamp(1); 414 Timestamp ts = rset.getTimestamp(1);
...@@ -421,14 +421,12 @@ public class OraMon { ...@@ -421,14 +421,12 @@ public class OraMon {
421 rset.close(); 421 rset.close();
422 422
423 //SYSSTATS 423 //SYSSTATS
424 if(colPdbSysStat == null) colPdbSysStat = new Collector(); 424 if(colPdbSysStat == null) colPdbSysStat = new Collector("colPdbSysStat");
425 colPdbSysStat.update(stmt.executeQuery("select systimestamp, c.NAME || '" + strSeparator + "' || s.name, s.value from V$CON_SYSSTAT s, V$CONTAINERS c where s.CON_ID=c.CON_ID")); 425 colPdbSysStat.update(stmt.executeQuery("select systimestamp, c.NAME || '" + strSeparator + "' || s.name, s.value from V$CON_SYSSTAT s, V$CONTAINERS c where s.CON_ID=c.CON_ID"));
426 } 426 }
427 427
428 stmt.close(); 428 stmt.close();
429 429
430 sanityCheck(pdbName);
431
432 getDataSucess++; 430 getDataSucess++;
433 lastFetchTSns = System.nanoTime(); 431 lastFetchTSns = System.nanoTime();
434 lastRTns = lastFetchTSns - startTSns; 432 lastRTns = lastFetchTSns - startTSns;
...@@ -452,30 +450,6 @@ public class OraMon { ...@@ -452,30 +450,6 @@ public class OraMon {
452 // End thread safe 450 // End thread safe
453 } 451 }
454 452
455 public void sanityCheck(String pdbName) throws Throwable {
456 String prefix = getFriendlyName(pdbName);
457 if(getBufferCacheHitRatioPercent(pdbName) < 0)
458 LOGGER.warning(prefix + " getBufferCacheHitRatioPercent" + " has negative delta");
459 if(getLogicalReadsPerSecond(pdbName) < 0)
460 LOGGER.warning(prefix + " getLogicalReadsPerSecond" + " has negative delta");
461 if(getCPUTimePerSecond(pdbName) < 0)
462 LOGGER.warning(prefix + " getCPUTimePerSecond" + " has negative delta");
463 if(getCacheHitRatioPercent(pdbName) < 0)
464 LOGGER.warning(prefix + " getCacheHitRatioPercent" + " has negative delta");
465 if(getCPUTimePerSecond(pdbName) < 0)
466 LOGGER.warning(prefix + " getCPUTimePerSecond" + " has negative delta");
467 if(getPerSecondValue("execute count",pdbName) < 0)
468 LOGGER.warning(prefix + " getPerSecondValue(execute count)" + " has negative delta");
469 if(getPerSecondValue("consistent gets",pdbName) < 0)
470 LOGGER.warning(prefix + " getPerSecondValue(consistent gets)" + " has negative delta");
471 if(getPerSecondValue("physical reads",pdbName) < 0)
472 LOGGER.warning(prefix + " getPerSecondValue(physical reads)" + " has negative delta");
473 if(getPerSecondValue("non-idle wait time",pdbName) < 0)
474 LOGGER.warning(prefix + " getPerSecondValue(non-idle wait time)" + " has negative delta");
475 if(getPerSecondValue("user commits",pdbName) < 0)
476 LOGGER.warning(prefix + " getPerSecondValue(user commits)" + " has negative delta");
477 }
478
479 public Throwable getLastEx() { 453 public Throwable getLastEx() {
480 return lastEx; 454 return lastEx;
481 } 455 }
......