Skip to content

Commit

Permalink
fixed shedThisRequest bug, fixed EDDGridFromErddap.sourceInfo bug, im…
Browse files Browse the repository at this point in the history 
…proved Math2.gc diagnostics

Former-commit-id: 6c030c5a461f49eba46854d2937a1b393380b38b [formerly 75c0208f3a8964821aad17ee5533ec8ddffa11fb] [formerly 03888715a39072a743a6d16cffdfbe58af10d7f8 [formerly ee1d3079d803b5fbaa26c88541dcba4d8a02618c]]
Former-commit-id: 43d528a08a82473bdf94629e970af46f46b28241 [formerly d6fb96acf8bca00b1b1f9921e5a6e1436ec8be8d]
Former-commit-id: 240fb59147ed8dcb98cb5fa9bbe14f60164be377
Former-commit-id: 4bb22ac672efa72e9097bf67faff5136c25a8275
  • Loading branch information
Bob Simons committed Dec 8, 2022
1 parent 07ff466 commit cb6bc44
Show file tree
Hide file tree
Showing 35 changed files with 161 additions and 135 deletions.
21 changes: 17 additions & 4 deletions WEB-INF/classes/com/cohort/array/ByteArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,19 +170,32 @@ public ByteArray(int first, int last) {
* @param fileName is the (usually canonical) path (dir+name) for the file
* @throws Exception if trouble
*/
public ByteArray(final String fileName) throws Exception {
this();
public static ByteArray fromFile(final String fileName) throws Exception {
ByteArray ba = new ByteArray();
final InputStream stream = File2.getDecompressedBufferedInputStream(fileName);
try {
int available = stream.available();
while (available > 0) {
ensureCapacity(size + (long)available);
size += stream.read(array, size, available);
ba.ensureCapacity(ba.size + (long)available);
ba.size += stream.read(ba.array, ba.size, available);
available = stream.available();
}
} finally {
stream.close();
}
return ba;
}

/**
* A constructor which makes a ByteArray from the UTF-8 bytes of the string.
*
* @param s a string. If null or 0-length, this returns a ByteArray with size=0.
* @throws Exception if trouble
*/
public static ByteArray fromString(final String s) throws Exception {
if (s == null || s.length() == 0)
return new ByteArray();
return new ByteArray(String2.stringToUtf8Bytes(s));
}

/** This constructs a ByteArray from the values of another PrimitiveArray by
Expand Down
9 changes: 5 additions & 4 deletions WEB-INF/classes/com/cohort/array/UByteArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,19 +212,20 @@ public UByteArray(final int first, final int last) {
* @param fileName is the (usually canonical) path (dir+name) for the file
* @throws Exception if trouble
*/
public UByteArray(final String fileName) throws Exception {
this();
public static UByteArray fromFile(final String fileName) throws Exception {
UByteArray uba = new UByteArray();
final InputStream stream = File2.getDecompressedBufferedInputStream(fileName);
try {
int available = stream.available();
while (available > 0) {
ensureCapacity(size + (long)available);
size += stream.read(array, size, available);
uba.ensureCapacity(uba.size + (long)available);
uba.size += stream.read(uba.array, uba.size, available);
available = stream.available();
}
} finally {
stream.close();
}
return uba;
}


Expand Down
8 changes: 4 additions & 4 deletions WEB-INF/classes/com/cohort/util/File2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ public static boolean delete(String fullName) {
);
if (attempt % 4 == 1)
//The problem might be that something needs to be gc'd.
Math2.gcAndWait(); //wait before retry delete. By experiment, gc works better than sleep.
Math2.gcAndWait("File2.delete (before retry)"); //By experiment, gc works better than sleep.
else Math2.sleep(1000);
}

Expand Down Expand Up @@ -639,7 +639,7 @@ public static void rename(String fullOldName, String fullNewName)

//failed? give it a second try. This fixed a problem in a test on Windows.
//The problem might be that something needs to be gc'd.
Math2.gcAndWait(); //wait before giving it a second try
Math2.gcAndWait("File2.rename (before retry)");
if (oldFile.renameTo(newFile))
return;
throw new RuntimeException("Unable to rename\n" + fullOldName + " to\n" + fullNewName);
Expand Down Expand Up @@ -797,7 +797,7 @@ public static long getLastModified(String fullName) {
//pause and try again
try {
//The problem might be that something needs to be gc'd.
Math2.gcAndWait(); //if trouble getting lastModified: gc encourages success
Math2.gcAndWait("File2.getLastModified (before retry)"); //if trouble getting lastModified: gc encourages success
File file = new File(fullName);
return file.lastModified();
} catch (Exception e2) {
Expand Down Expand Up @@ -1319,7 +1319,7 @@ public static String[] readFromFile(String fileName, String charset, int maxAtte
String2.log("WARNING #" + attempt +
": File2.readFromFile is having trouble. It will try again to read " +
fileName);
if (attempt == 1) Math2.gcAndWait(); //trouble! Give OS/Java a time and gc to deal with trouble
if (attempt == 1) Math2.gcAndWait("File2.readFromFile (before retry)"); //trouble! Give OS/Java a time and gc to deal with trouble
else Math2.sleep(1000);
}
}
Expand Down
24 changes: 16 additions & 8 deletions WEB-INF/classes/com/cohort/util/Math2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public class Math2 {
public static long dangerousMemory = maxMemory * 9L / 10; //90% this is really bad

public static long alwaysOkayMemoryRequest = maxSafeMemory / 40;
public volatile static int gcCallCount = 0; //since last Major LoadDatasets
public volatile static long timeGCLastCalled = 0;

/**
Expand Down Expand Up @@ -381,9 +382,10 @@ public static String xmxMemoryString() {
* It will call gc() (not incgc) if memory use is creeping up.
* 2013-12-05 Years ago, I called this often. But now Java recommends letting Java handle memory/gc.
*
* @param caller for diagnostics, the name of the caller
* @param millis the number of millis to sleep
*/
public static void incgc(final long millis) {
public static void incgc(final String caller, final long millis) {
//long time = System.currentTimeMillis(); //diagnostic

//get usingMemory
Expand All @@ -400,7 +402,7 @@ public static void incgc(final long millis) {
// how tight I want it to be
//smaller numbers will cause more gc's
} else if ((using - lastUsingMemory) > gcTrigger) {
gc(millis); //this will also sleep or yield
gc(caller, millis); //this will also sleep or yield

//intermediate, just sleep
} else {
Expand All @@ -416,10 +418,11 @@ public static void incgc(final long millis) {
* This calls gc(shortSleep).
* shortSleep is intended to give gc sufficient time to do its job, even under heavy use.
*
* @param caller for diagnostics, the name of the caller
* @return getMemoryInUse()
*/
public static long gcAndWait() {
return gc(shortSleep);
public static long gcAndWait(final String caller) {
return gc(caller, shortSleep);
}

/**
Expand All @@ -432,24 +435,29 @@ public static long gcAndWait() {
* will mind, since it is static) and to sleep for
* a while, independently.
*
* @param caller for diagnostics, the name of the caller.
* The 2 important callers are shedThisRequest (before an ERDDAP request is processed)
* and ensureMemoryAvailable (can occur anytime).
* @param millis the number of milliseconds to sleep.
* This will always sleep at least shortSleep millis.
* @return getMemoryInUse()
* @see Math2#incgc
*/
public static long gc(final long millis) {
public static long gc(final String caller, final long millis) {
final long time = System.currentTimeMillis();

//if >shortSleep since last time gc was called (by any thread), call gc (otherwise don't!)
if (time - timeGCLastCalled >= shortSleep) {
String2.log("[Math2.gc is calling System.gc]");
String2.log("[Math2.gc called by " + caller +
" at " + Calendar2.getCurrentISODateTimeStringLocalTZ() +
" (call #" + ++gcCallCount + " since last major loadDatasets)]");
//+ "\n" + MustBe.getStackTrace());
timeGCLastCalled = time;
System.gc();
}

//sleep - subtract time already used by gc
//always call sleep, even if <0, since it yields, too.
//always wait at least shortSleep so we can see the effects of the gc
sleep(Math.max(shortSleep, millis) - (System.currentTimeMillis() - time));
return lastUsingMemory = getMemoryInUse();
}
Expand Down Expand Up @@ -491,7 +499,7 @@ public static void ensureMemoryAvailable(final long nBytes, final String attribu
return;

//Not enough memory. Try to free up memory
memoryInUse = Math2.gcAndWait(); //if gc just called by other thread, this just waits and returns
memoryInUse = Math2.gcAndWait("ensureMemoryAvailable (" + attributeTo + ")"); //if gc just called by other thread, this just waits and returns

//now enough memory?
if (memoryInUse + nBytes <= maxSafeMemory) //it'll work
Expand Down
2 changes: 1 addition & 1 deletion WEB-INF/classes/com/cohort/util/String2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3763,7 +3763,7 @@ public static void lowLog(String message, boolean addNewline) {
logFile = File2.getBufferedFileWriterUtf8(logFileName);
} catch (Throwable t) {
//try again: really bad if unable to create a new logFile
Math2.gc(1000);
Math2.gc("String2.lowLog", 1000);
try {
logFile = File2.getBufferedFileWriterUtf8(logFileName);
} catch (Throwable t2) {
Expand Down
48 changes: 24 additions & 24 deletions WEB-INF/classes/com/cohort/util/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,15 +287,15 @@ public static void testMath2() {
//incgc
String2.log("test incgc(3000)");
da = null; //free the memory
Math2.incgc(3000);
Math2.incgc("EDDTableFromNcFiles (between tests)", 3000);
String2.log("after incgc: " + Math2.memoryString());

//gc
String2.log("test gcAndWait()");
da = new double[1000000];
String2.log("after allocate = " + Math2.memoryString());
da = null;
Math2.gcAndWait(); //in test
Math2.gcAndWait("TestUtil (between tests)"); //in test
String2.log("after gc = " + Math2.memoryString());

//odd
Expand Down Expand Up @@ -1242,7 +1242,7 @@ public static void testString2() throws Throwable {
int n, iar[];
String s, results, expected;
long time;
/*
/* */
//clipboard
String2.log("Clipboard was: " + String2.getClipboardString());
String2.setClipboardString("Test String2.setClipboardString.");
Expand All @@ -1260,7 +1260,7 @@ public static void testString2() throws Throwable {
t1 += String2.parseInt("1023456789");
String2.log("time1 for 1000000 parseInt good=" + (System.currentTimeMillis() - time)); //119

//speed of parseInt fail
//speed of parseInt fail
time = System.currentTimeMillis();
long t2 = 0;
for (int i = 0; i < 1000000; i++)
Expand Down Expand Up @@ -1512,7 +1512,7 @@ public static void testString2() throws Throwable {
//if (test == 10) System.exit(0);
}

Math2.incgc(3000);
Math2.incgc("EDDTableFromNcFiles (between tests)", 3000);

//binaryFindLastLE
String tsar[] = {"abc", "bcd", "bcj"};
Expand Down Expand Up @@ -2544,11 +2544,11 @@ public static void testString2() throws Throwable {
Test.ensureEqual(String2.parseLong("0xFFFFFFFFFF"), 0xFFFFFFFFFFL, "g");

//test speed of native parseLong vs String2.parseLong (now using parseBigDecimal)
int n = 10000000;
n = 10000000;
long speedResults[] = new long[2];
for (int test = 0; test < 2; test++) {
Math2.gc(2000);
Math2.gc(2000);
Math2.gc("TestUtil (between tests)", 2000);
Math2.gc("TestUtil (between tests)", 2000);
long testSum = 0;
time = System.currentTimeMillis();
for (i = 0; i < n; i++) {
Expand All @@ -2564,8 +2564,8 @@ public static void testString2() throws Throwable {
String2.toCSSVString(speedResults));
Test.ensureTrue(speedResults[1] < speedResults[0] * 2,
"String2.parseLong is too slow! " + speedResults[1] + " vs " + speedResults[0] + " (Java 17 typical: 1900ms vs 1300ms");
Math2.gc(2000);
Math2.gc(2000);
Math2.gc("TestUtil (between tests)", 2000);
Math2.gc("TestUtil (between tests)", 2000);

//parseFloat
String2.log("test parseFloat");
Expand All @@ -2578,7 +2578,7 @@ public static void testString2() throws Throwable {
//tokenize
String2.log("test tokenize");
Test.ensureEqual(String2.tokenize(" a bb ccc \"d d\" eeee"), new String[]{"a", "bb", "ccc", "d d", "eeee"}, "a");
*/

//distributeTime and timeDistributionStatistics
String2.log("test distributeTime");
int timeDist[] = new int[String2.TimeDistributionSize];
Expand Down Expand Up @@ -6015,7 +6015,7 @@ public static void testCalendar2() throws Throwable {
Test.ensureEqual(Calendar2.epochSecondsToLimitedIsoStringT(
"1970-01-01T00:00:00.000Z", d, "."), "-0005-08-31T16:01:02.123Z", "");

Math2.gcAndWait(); //in test
Math2.gcAndWait("TestUtil (between tests)"); //in test
}

/**
Expand Down Expand Up @@ -6163,8 +6163,8 @@ public static void testFile2() throws Exception {

//test boolean touch(String dirName) and getLastModified
String2.log("test touch and getLastModified");
Math2.gc(1000);
Math2.gc(1000);
Math2.gc("TestUtil (between tests)", 1000);
Math2.gc("TestUtil (between tests)", 1000);
File2.writeToFile88591(utilDir + "temp.txt", "This\nis a\n\ntest.\n");
Math2.sleep(20); //make the file a little older
long fileTime = File2.getLastModified(utilDir + "temp.txt");
Expand Down Expand Up @@ -6463,14 +6463,14 @@ public static void testString2utf8() throws Exception {

for (int loop = 0; loop < 3; loop++) {
//test Strings
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
memoryInUse = Math2.getMemoryInUse();
time = System.currentTimeMillis();
String sa[] = new String[n];
for (int i = 0; i < n; i++)
sa[i] = "testABCD" + i;
time = System.currentTimeMillis() - time;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
String2.log("String memoryUse/item=" +
((Math2.getMemoryInUse() - memoryInUse) / (n + 0.0)) + //68.1 bytes
" time=" + time + "ms"); // ~562 (after first time)
Expand All @@ -6482,7 +6482,7 @@ public static void testString2utf8() throws Exception {
for (int i = 0; i < n; i++)
ba[i] = String2.stringToUtf8Bytes("testABCD" + i); //usually 14 characters +4length +4pointer
time = System.currentTimeMillis() - time;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
String2.log("utf8 memoryUse/item=" +
((Math2.getMemoryInUse() - memoryInUse) / (n + 0.0)) + //36.0 bytes; why so many?
" time=" + time + "ms"); // ~1094
Expand All @@ -6494,7 +6494,7 @@ public static void testString2utf8() throws Exception {
for (int i = 0; i < n; i++)
da[i] = i;
time = System.currentTimeMillis() - time;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
String2.log("double memoryUse/item=" +
((Math2.getMemoryInUse() - memoryInUse) / (n + 0.0)) + //8 bytes
" time=" + time + "ms"); // ~6
Expand Down Expand Up @@ -6529,7 +6529,7 @@ public static void testString2canonical() throws Exception {
int n = 1000;
String sa[] = new String[95*95];
long oMemoryInUse = -1;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive preparation //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive preparation //in a test
String2.log("initialMemoryUse=" + Math2.memoryString() + "\n" +
String2.canonicalStatistics());
int canSize = -1;
Expand Down Expand Up @@ -6561,7 +6561,7 @@ public static void testString2canonical() throws Exception {

//ensure that memory use and nStrings in maps don't grow unexpectedly
time = System.currentTimeMillis() - time;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
long memoryInUse = Math2.getMemoryInUse();
int shouldBe = outer == 0? 415 : 260;
String2.log(String2.canonicalStatistics() +
Expand Down Expand Up @@ -6606,7 +6606,7 @@ public static void testString2canonicalStringHolder() throws Exception {
int n = 1000;
StringHolder sa[] = new StringHolder[95*95];
long oMemoryInUse = -1;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive preparation //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive preparation //in a test
String2.log("initialMemoryUse=" + Math2.memoryString() + "\n" +
String2.canonicalStatistics());
int canSize = -1;
Expand Down Expand Up @@ -6638,7 +6638,7 @@ public static void testString2canonicalStringHolder() throws Exception {

//ensure that memory use and nStrings in maps don't grow unexpectedly
time = System.currentTimeMillis() - time;
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
long memoryInUse = Math2.getMemoryInUse();
int shouldBe = outer == 0? 415 : 260; //ms
String2.log(String2.canonicalStatistics() +
Expand Down Expand Up @@ -6678,7 +6678,7 @@ public static void testString2canonical2() throws Exception {
String sar[] = new String[127];

//what is initial memory level?
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
long oMem = Math2.getMemoryInUse();

//just store first 5 chars large strings
Expand All @@ -6688,7 +6688,7 @@ public static void testString2canonical2() throws Exception {
}

//what is final memory level?
Math2.gcAndWait(); Math2.gcAndWait(); //aggressive //in a test
Math2.gcAndWait("TestUtil (between tests)"); Math2.gcAndWait("TestUtil (between tests)"); //aggressive //in a test
long cMem = Math2.getMemoryInUse();
String2.log("oMem=" + oMem + "\n" +
"cMem=" + cMem + "\n");
Expand Down
2 changes: 1 addition & 1 deletion WEB-INF/classes/gov/noaa/pfel/coastwatch/Browser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ private String getDataFromNextShared() {
entrySetIterator = null;

//generate report of active datasets in nextShared
Math2.gcAndWait(); Math2.gcAndWait(); //for more accurate memoryString in report below
Math2.gcAndWait("Browser"); Math2.gcAndWait("Browser"); //for more accurate memoryString in report below
StringBuilder activeDataSets = new StringBuilder();
activeDataSets.append(
"Report from " + oneOf.shortClassName() + " at " +
Expand Down
Loading

0 comments on commit cb6bc44

Please sign in to comment.