From 54a02393d771e7cebb9d3b7be4dd4340662ea0e7 Mon Sep 17 00:00:00 2001 From: Bob Simons Date: Fri, 30 Sep 2022 13:44:10 -0400 Subject: [PATCH] v2.20 Former-commit-id: a9a5385df1fe576c89a5a3d0bc39ae2207f849a5 [formerly cfc7175ab3e87358f028b022315eeccaf64184a9] [formerly 30ad470db93301fad6f2a667accba220f20223eb [formerly 9f47d1a59bcb23e9b074968dde31ce75afa6497c]] Former-commit-id: a1b0610dd1cce337b682314e85f5f3976b1873b1 [formerly fbe1d2f4bb405b7dcb34f0a3757db8b307afff42] Former-commit-id: e835167398bc18970028afcb36c526d893d4f8d3 Former-commit-id: 809cc95258ea0a1b558c9699fde9b2925d219c94 --- WEB-INF/classes/com/cohort/util/Math2.java | 4 +- .../classes/gov/noaa/pfel/erddap/Erddap.java | 4 +- .../gov/noaa/pfel/erddap/LoadDatasets.java | 9 ++- .../gov/noaa/pfel/erddap/util/EDStatic.java | 42 +++++++------- download/changes.html | 50 +++++++++++++++-- download/setup.html | 55 +++++++++++++------ download/setupDatasetsXml.html | 4 +- 7 files changed, 118 insertions(+), 50 deletions(-) diff --git a/WEB-INF/classes/com/cohort/util/Math2.java b/WEB-INF/classes/com/cohort/util/Math2.java index 7c56d20ea..043a046ce 100644 --- a/WEB-INF/classes/com/cohort/util/Math2.java +++ b/WEB-INF/classes/com/cohort/util/Math2.java @@ -65,7 +65,7 @@ public class Math2 { public static long maxSafeMemory = maxMemory * 3L / 4; //75% the max we should consider getting to public static long dangerousMemory = maxMemory * 9L / 10; //90% this is really bad - public static long alwaysOkayMemoryRequest = maxSafeMemory / 20; + public static long alwaysOkayMemoryRequest = maxSafeMemory / 40; /** * These are *not* final so EDStatic can replace them with translated Strings. @@ -468,7 +468,7 @@ public static void ensureMemoryAvailable(final long nBytes, final String attribu // even if it was for ERDDAP management (i.e., shooting myself in the foot). //this is a little risky, but avoids frequent calls to calculate memoryInUse - if (nBytes < alwaysOkayMemoryRequest) //e.g., 8GB -> maxSafe=6GB /20=300MB + if (nBytes < alwaysOkayMemoryRequest) //e.g., 8GB -> maxSafe=6GB /40=150MB return; //is this single request by itself too big under any circumstances? diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java index bc88c2ec1..30124fa94 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java @@ -3610,7 +3610,9 @@ public void doStatus(int language, HttpServletRequest request, HttpServletRespon int po = traces.indexOf('\n'); if (po > 0) sb.append(traces.substring(0, po + 1)); - sb.append(EDStatic.gcCalled + " gc calls and " + EDStatic.requestsShed + " requests shed since last major LoadDatasets\n"); + sb.append(EDStatic.gcCalled + " gc calls, " + + EDStatic.requestsShed + " requests shed, and " + + EDStatic.dangerousMemoryEmails + " dangerousMemoryEmails since last major LoadDatasets\n"); sb.append(Math2.memoryString() + " " + Math2.xmxMemoryString() + "\n\n"); EDStatic.addCommonStatistics(sb); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java b/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java index 72791073c..657670273 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java @@ -1055,6 +1055,7 @@ public void run() { //reset since last majorReload EDStatic.gcCalled = 0; EDStatic.requestsShed = 0; + EDStatic.dangerousMemoryEmails = 0; EDStatic.dangerousMemoryFailures = 0; EDStatic.tooManyRequests = 0; @@ -1077,7 +1078,9 @@ public void run() { if (threadSummary != null) contentSB.append(threadSummary + "\n"); - contentSB.append(EDStatic.gcCalled + " gc calls and " + EDStatic.requestsShed + " requests shed since last major LoadDatasets\n"); + contentSB.append(EDStatic.gcCalled + " gc calls, " + + EDStatic.requestsShed + " requests shed, and " + + EDStatic.dangerousMemoryEmails + " dangerousMemoryEmails since last major LoadDatasets\n"); contentSB.append(Math2.memoryString() + " " + Math2.xmxMemoryString() + "\n\n"); contentSB.append(stars + "\nTallied Usage Information\n\n"); contentSB.append(EDStatic.tally.toString(50)); @@ -1215,7 +1218,9 @@ public void run() { if (threadSummary != null) sb.append(threadSummary + "\n"); - sb.append(EDStatic.gcCalled + " gc calls and " + EDStatic.requestsShed + " requests shed since last major LoadDatasets\n"); + sb.append(EDStatic.gcCalled + " gc calls, " + + EDStatic.requestsShed + " requests shed, and " + + EDStatic.dangerousMemoryEmails + " dangerousMemoryEmails since last major LoadDatasets\n"); sb.append(Math2.memoryString() + " " + Math2.xmxMemoryString() + "\n\n"); EDStatic.addCommonStatistics(sb); sb.append(EDStatic.tally.toString("Large Request, IP address (since last Major LoadDatasets)", 50)); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java index 89b80b49e..1d6becd91 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java @@ -186,7 +186,7 @@ public class EDStatic { *
2.17 released on 2022-02-16 *
2.18 released on 2022-02-23 *
2.19 released on 2022-09-10 - *
2.20 released on 2022-09-?? + *
2.20 released on 2022-09-30 * * For master branch releases, this will be a floating point * number with 2 decimal digits, with no additional text. @@ -265,6 +265,7 @@ public class EDStatic { public static int touchThreadSucceededDistributionTotal[]= new int[String2.TimeDistributionSize]; public static volatile int gcCalled = 0; //since last Major LoadDatasets public static volatile int requestsShed = 0; //since last Major LoadDatasets + public static volatile int dangerousMemoryEmails = 0; //since last Major LoadDatasets public static volatile int dangerousMemoryFailures = 0; //since last Major LoadDatasets public static StringBuffer suggestAddFillValueCSV = new StringBuffer(); //EDV constructors append message here //thread-safe but probably doesn't need to be @@ -288,12 +289,12 @@ public class EDStatic { public final static String ipAddressUnknown = "(unknownIPAddress)"; public final static ConcurrentHashMap ipAddressQueue = new ConcurrentHashMap(); //ipAddress -> list of request# public final static int DEFAULT_ipAddressMaxRequestsActive = 2; //in datasets.xml - public final static int DEFAULT_ipAddressMaxRequests = 7; //in datasets.xml //more requests will see Too Many Requests error. This must be at least 6 because browsers make up to 6 simultaneous requests. This can't be >1000. + public final static int DEFAULT_ipAddressMaxRequests = 15; //in datasets.xml //more requests will see Too Many Requests error. This must be at least 6 because browsers make up to 6 simultaneous requests. This can't be >1000. public final static String DEFAULT_ipAddressUnlimited = ", " + ipAddressUnknown; public static int ipAddressMaxRequestsActive = DEFAULT_ipAddressMaxRequestsActive; //in datasets.xml public static int ipAddressMaxRequests = DEFAULT_ipAddressMaxRequests; //in datasets.xml //more requests will see Too Many Requests error. This must be at least 6 because browsers make up to 6 simultaneous requests. public static HashSet ipAddressUnlimited = //in datasets.xml //read only. New one is swapped into place. You can add and remove addresses as needed. - new HashSet(String2.toArrayList(StringArray.fromCSVNoBlanks(EDStatic.DEFAULT_ipAddressUnlimited).toArray())); + new HashSet(String2.toArrayList(StringArray.fromCSVNoBlanks(DEFAULT_ipAddressUnlimited).toArray())); public static int tooManyRequests = 0; //nRequests exceeding ipAddressMaxRequests, since last major datasets reload public final static String translationDisclaimer = //from https://cloud.google.com/translate/attribution @@ -2514,7 +2515,7 @@ else if ((upo != -1 && upo != 4) || convertUnitsNotesAr = getNotNothingString(messagesAr, "convertUnitsNotes", errorInMethod); for (int tl = 0; tl < nLanguages; tl++) convertUnitsNotesAr[tl] = convertUnitsNotesAr[tl] - .replace("&unitsStandard;", EDStatic.units_standard); + .replace("&unitsStandard;", units_standard); convertUnitsServiceAr = getNotNothingString(messagesAr, "convertUnitsService", errorInMethod); convertURLsAr = getNotNothingString(messagesAr, "convertURLs", errorInMethod); convertURLsIntroAr = getNotNothingString(messagesAr, "convertURLsIntro", errorInMethod); @@ -3497,7 +3498,7 @@ else if ((upo != -1 && upo != 4) || WMSNotesAr = getNotNothingString(messagesAr, "WMSNotes", errorInMethod); for (int tl = 0; tl < nLanguages; tl++) - EDStatic.WMSNotesAr[tl] = EDStatic.WMSNotesAr[tl].replace( "&WMSSEPARATOR;", Character.toString(EDD.WMS_SEPARATOR)); + WMSNotesAr[tl] = WMSNotesAr[tl].replace( "&WMSSEPARATOR;", Character.toString(EDD.WMS_SEPARATOR)); yourEmailAddressAr = getNotNothingString(messagesAr, "yourEmailAddress", errorInMethod); zoomInAr = getNotNothingString(messagesAr, "zoomIn", errorInMethod); @@ -4677,8 +4678,8 @@ public static void addCommonStatistics(StringBuilder sb) { sb.append(String2.getTimeDistributionStatistics(touchThreadSucceededDistributionTotal)); sb.append('\n'); sb.append('\n'); - sb.append(EDStatic.tally.toString("Language (since last daily report)", 50)); //added v2.15 - sb.append(EDStatic.tally.toString("Language (since startup)", 50)); + sb.append(tally.toString("Language (since last daily report)", 50)); //added v2.15 + sb.append(tally.toString("Language (since startup)", 50)); sb.append(SgtMap.topographyStats() + "\n"); sb.append(GSHHS.statsString() + "\n"); @@ -4956,15 +4957,15 @@ public static void sendHttpUnauthorizedError(int language, int requestNumber, St if (datasetID != null && datasetID.length() > 0) message = MessageFormat.format( graphsAccessibleToPublic? - EDStatic.notAuthorizedForDataAr[language] : - EDStatic.notAuthorizedAr[language], + notAuthorizedForDataAr[language] : + notAuthorizedAr[language], loggedInAsHttps.equals(loggedInAs)? "" : loggedInAs, datasetID); lowSendError(requestNumber, response, HttpServletResponse.SC_UNAUTHORIZED, message); } catch (Throwable t2) { - EDStatic.rethrowClientAbortException(t2); //first thing in catch{} + rethrowClientAbortException(t2); //first thing in catch{} String2.log("Error in sendHttpUnauthorizedError for request #" + requestNumber + ":\n" + (message == null? "" : message + "\n") + MustBe.throwableToString(t2)); @@ -5984,7 +5985,7 @@ public static String passThroughJsonpQuery(int language, HttpServletRequest requ return ""; String functionName = jsonp.substring(7); //it will be because it starts with .jsonp= if (!String2.isJsonpNameSafe(functionName)) - throw new SimpleException(EDStatic.bilingual(language, + throw new SimpleException(bilingual(language, errorJsonpFunctionNameAr[0], errorJsonpFunctionNameAr[language])); return ".jsonp=" + SSR.minimalPercentEncode(functionName) + "&"; @@ -6152,7 +6153,7 @@ public static String nMatchingDatasetsHtml(int language, int nMatches, int page, "...\n"); if (page >= 3) sb.append( url0 + (page>2? prev : bmrk) + url1 + (page - 1) + url2 + (page - 1) + url3); - sb.append(" " + page + " (" + EDStatic.nMatchingCurrentAr[language] + ") \n"); //always show current page + sb.append(" " + page + " (" + nMatchingCurrentAr[language] + ") \n"); //always show current page if (page <= lastPage - 2) sb.append( url0 + (page= Math2.dangerousMemory && lastActiveRequestReportTime == 0) { lastActiveRequestReportTime = System.currentTimeMillis(); //do first so other threads don't also report this + dangerousMemoryEmails++; activeRequests.remove(requestNumber + ""); //don't blame this request String activeRequestLines[] = activeRequests.values().toArray(new String[0]); Arrays.sort(activeRequestLines); @@ -6449,7 +6451,7 @@ public static boolean shedThisRequest(int language, int requestNumber, HttpServl "Active requests:\n" + String2.toNewlineString(activeRequestLines); String2.log(report); - email(EDStatic.emailEverythingToCsv, "Dangerously High Memory Use!!!", report); + email(emailEverythingToCsv, "Dangerously High Memory Use!!!", report); } //memory use is too high, so shed this request @@ -6577,7 +6579,7 @@ public static void sendError(int requestNumber, HttpServletRequest request, } else if (tError.indexOf(Math2.memoryArraySize.substring(0, 25)) >= 0) { errorNo = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; //http error 413 (the old name for Payload Too Large), although it could be other user's requests that are too large - String ipAddress = EDStatic.getIPAddress(request); + String ipAddress = getIPAddress(request); tally.add("OutOfMemory (Array Size), IP Address (since last Major LoadDatasets)", ipAddress); tally.add("OutOfMemory (Array Size), IP Address (since last daily report)", ipAddress); tally.add("OutOfMemory (Array Size), IP Address (since startup)", ipAddress); @@ -6586,7 +6588,7 @@ public static void sendError(int requestNumber, HttpServletRequest request, tError.indexOf(Math2.memoryThanCurrentlySafe.substring(0, 25)) >= 0) { //!!! TROUBLE: but that matches memoryThanSafe (in English) too! errorNo = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; //http error 413 (the old name for Payload Too Large), although it could be other user's requests that are too large dangerousMemoryFailures++; - String ipAddress = EDStatic.getIPAddress(request); + String ipAddress = getIPAddress(request); tally.add("OutOfMemory (Too Big), IP Address (since last Major LoadDatasets)", ipAddress); tally.add("OutOfMemory (Too Big), IP Address (since last daily report)", ipAddress); tally.add("OutOfMemory (Too Big), IP Address (since startup)", ipAddress); @@ -6594,7 +6596,7 @@ public static void sendError(int requestNumber, HttpServletRequest request, } else if (tErrorLC.indexOf(Math2.memory) >= 0) { //catchall for remaining memory problems errorNo = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; //http error 413 (the old name for Payload Too Large) - String ipAddress = EDStatic.getIPAddress(request); + String ipAddress = getIPAddress(request); tally.add("OutOfMemory (Way Too Big), IP Address (since last Major LoadDatasets)", ipAddress); tally.add("OutOfMemory (Way Too Big), IP Address (since last daily report)", ipAddress); tally.add("OutOfMemory (Way Too Big), IP Address (since startup)", ipAddress); @@ -6658,8 +6660,8 @@ public static void lowSendError(int requestNumber, HttpServletResponse response, //because any of these errors could be in a script //and it's good to slow the script down (prevent 100 bad requests/second) //and if it's a human they won't even notice a short delay - if (EDStatic.slowDownTroubleMillis > 0) - Math2.sleep(EDStatic.slowDownTroubleMillis); + if (slowDownTroubleMillis > 0) + Math2.sleep(slowDownTroubleMillis); //put the HTTP status code name at the start of the message (from Wikipedia list // https://en.wikipedia.org/wiki/List_of_HTTP_status_codes @@ -6743,7 +6745,7 @@ public static void testUpdateUrls() throws Exception { add.set( "b", "http://www.whoi.edu"); //purposely out-of-date add.set( "ten", 10.0); add.set( "sourceUrl", "http://coastwatch.pfel.noaa.gov"); //purposely out-of-date - EDStatic.updateUrls(source, add); + updateUrls(source, add); String results = add.toString(); String expected = " a=https://coastwatch.pfeg.noaa.gov\n" + @@ -6758,7 +6760,7 @@ public static void testUpdateUrls() throws Exception { add.set("b", "http://www.whoi.edu"); add.set("nine", 9.0); add.set("sourceUrl", "http://coastwatch.pfel.noaa.gov"); - EDStatic.updateUrls(null, add); + updateUrls(null, add); results = add.toString(); expected = " a=https://coastwatch.pfeg.noaa.gov\n" + diff --git a/download/changes.html b/download/changes.html index 53ad2fafd..aa8a42308 100644 --- a/download/changes.html +++ b/download/changes.html @@ -68,11 +68,11 @@

ERDDAP Changes

--> - -

Changes -in ERDDAP version 2.20 (released 2022-09-01)

+

Changes +in ERDDAP version 2.20 (released 2022-09-30)

@@ -92,6 +129,9 @@

Chan

Changes in ERDDAP version 2.19 (released 2022-09-01)

    +
  • Don't use v2.19. It is flawed. But administrators still need to do the TO DO + items listed below when upgrading to v2.20+. +
     
  • New Features and Changes (for users):
    • NEW: There is a new server-side function, orderByDescending, which works like diff --git a/download/setup.html b/download/setup.html index 6acfeba27..ea877a6ad 100644 --- a/download/setup.html +++ b/download/setup.html @@ -521,24 +521,17 @@

        - -
    • Resources Cache -
      To avoid Tomcat throwing warnings during startup about - "org.apache.catalina.webresources.Cache.getResource Unable to add the resource - at someFile to the cache because there was insufficient free space - available after evicting expired cache entries -- consider increasing the - maximum size of the cache: -
      In your [tomcat]/conf/context.xml, add this tag right before </Context> : - -
      <Resources cachingAllowed="true" cacheMaxSize="80000" /> -
      (or some other number, in KB.) For more information see -
      https://tomcat.apache.org/tomcat-8.0-doc/config/resources.html (external link). -
        +
    • Resources Cache - + In tomcat/conf/context.xml, right before the </Context> tag, + change the Resources tag (or add it if it isn't already there) to + increase the cacheMaxSize setting from 80000 (or whatever it was) to 200000: +
      <Resources cachingAllowed="true" cacheMaxSize="200000" /> +
      This avoids numerous warnings in catalina.out that all start with +
      "WARNING [main] org.apache.catalina.webresources.Cache.getResource Unable to add the resource at [/WEB-INF/classes/..." +
        +
    • On Linux computers, change the Apache timeout settings so that time-consuming user requests don't timeout @@ -2435,6 +2428,32 @@

      Things
       

    +
  • Memory Status +
    ERDDAP shouldn't ever crash or freeze up. If it does, one of the + most likely causes is insufficient memory. You can monitor memory usage + by looking at the status.html web page, which includes a line like +
    0 gc calls, 0 requests shed, and 0 dangerousMemoryEmails since last major LoadDatasets +
    (those are progressively more serious events) +
    and MB inUse and gc Calls columns in the table of statistics. + You can tell how memory-stressed your ERDDAP is by watching these numbers. + Higher numbers indicate more stress. +
      +
    • MB inUse should always be less than half of the + -Xmx memory setting. + Larger numbers are serious trouble. +
    • gc calls indicates the number of times ERDDAP called the + garbage collector to try to alleviate high memory usage. +
    • shed indicates the number of incoming requests that + were shed (with HTTP error number 503, Service Unavailable) because + memory use was already too high. Ideally, no requests should be shed. +
    • dangerousMemoryEmails - + If memory use becomes dangerously high, ERDDAP sends an email to the email addresses listed in + emailEverythingToCSV (in setup.xml) with a list of the active user requests. As the email says, + please forward these emails to erd.data at noaa.gov so we can use the information + to improve future versions of ERDDAP. +
    +
  • OutOfMemoryError
    When you set up ERDDAP, you specify the maximum amount of memory that Java @@ -2530,7 +2549,7 @@

    Things file to see what ERDDAP was doing when the error arose, you can usually get a good clue as to the cause. In most cases, there is a way to minimize that problem (see above), -but sometimes you just need more memory. +but sometimes you just need more memory and a higher -Xmx setting.
     

diff --git a/download/setupDatasetsXml.html b/download/setupDatasetsXml.html index ad783439a..58c54f8af 100644 --- a/download/setupDatasetsXml.html +++ b/download/setupDatasetsXml.html @@ -10702,7 +10702,7 @@

Details< requests will receive an HTTP 429 error: Too Many Requests. The small, static files in erddap/download/ and erddap/images/ are NOT exempt from this count. - The default is 7. The maximum allowed is 1000, which crazy high -- don't do it! + The default is 15. The maximum allowed is 1000, which is crazy high -- don't do it! ERDDAP won't accept a number less than 6 because many legitimate users (notably web browsers and WMS clients) make up to 6 requests at a time. The ERDDAP Daily Report and the @@ -10740,7 +10740,7 @@

Details< have been processed. The small, static files in erddap/download/ and erddap/images/ ARE exempt from this count and the related throttling. - The default is 2. The maximum allowed is 100, which crazy high -- don't do it! + The default is 2. The maximum allowed is 100, which is crazy high -- don't do it! You can set this to 1 to be strict, especially if you have problems with overly aggressive or malicious users. Users will still quickly get all the data they request (up to ipAddressMaxRequests),