diff --git a/bin/mv_batch.sh b/bin/mv_batch.sh index ff048151..246fc011 100755 --- a/bin/mv_batch.sh +++ b/bin/mv_batch.sh @@ -16,19 +16,19 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" MV_HOME="$(dirname "$DIR")" # construct the classpath for MVBatch -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xml-apis.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/guava-33.4.8.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/servlet-api.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/jakarta.servlet-api-5.0.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.5.jar CLASSPATH=$CLASSPATH:$MV_HOME/dist/lib/metviewer.jar PYTHON_ENV= diff --git a/bin/mv_compare.sh b/bin/mv_compare.sh index 22c8acb7..d40e893c 100755 --- a/bin/mv_compare.sh +++ b/bin/mv_compare.sh @@ -71,27 +71,28 @@ fi CLASSPATH=${MV_HOME}/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/xml-apis.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/guava-33.4.8.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/junit-4.11.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/hamcrest-core-1.3.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-io-2.20.0.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/mockito-all-1.9.5.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/servlet-api-4.0.1.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload-1.6.0.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/jakarta.servlet-api-5.0.0.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload2-core-2.0.0-M4.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload2-jakarta-servlet5-2.0.0-M4.jar CLASSPATH=$CLASSPATH:${MV_HOME}/dist/lib/metviewer_all.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/j2html-1.4.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-core-2.20.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-databind-2.20.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.11.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.19.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.5.jar echo "Running allRestRunner" #/Users/pierce/test_data all diff --git a/bin/mv_load_mvload.sh b/bin/mv_load_mvload.sh index 51a0f26c..2a74c493 100755 --- a/bin/mv_load_mvload.sh +++ b/bin/mv_load_mvload.sh @@ -19,18 +19,18 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" MV_HOME="$(dirname "$DIR")" # construct the classpath for MVLoad -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xml-apis.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/guava-33.4.8.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/servlet-api.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/jakarta.servlet-api-5.0.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/dist/lib/metviewer.jar java -classpath $CLASSPATH -Xmx2048M edu.ucar.metviewer.MVLoad $@ diff --git a/bin/mv_prune.sh b/bin/mv_prune.sh index cb3f6375..7be655b2 100755 --- a/bin/mv_prune.sh +++ b/bin/mv_prune.sh @@ -17,18 +17,18 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" MV_HOME="$(dirname "$DIR")" # construct the classpath for MVPruneDB -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xml-apis.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/guava-33.4.8.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/servlet-api.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/jakarta.servlet-api-5.0.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/dist/lib/metviewer.jar java -classpath $CLASSPATH -Xmx2048M edu.ucar.metviewer.prune.MVPruneDB $@ \ No newline at end of file diff --git a/bin/mv_scorecard.sh b/bin/mv_scorecard.sh index f6adaea4..1f98d25a 100755 --- a/bin/mv_scorecard.sh +++ b/bin/mv_scorecard.sh @@ -17,23 +17,23 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" MV_HOME="$(dirname "$DIR")" # construct the classpath for MVBatch -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/xml-apis.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/guava-33.4.8.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/j2html-1.4.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-core-2.20.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-databind-2.20.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.11.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/servlet-api.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.19.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/jakarta.servlet-api-5.0.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.5.jar CLASSPATH=$CLASSPATH:$MV_HOME/dist/lib/metviewer.jar PYTHON_ENV= diff --git a/bin/mv_test.sh b/bin/mv_test.sh index 8507aa55..4d4e8749 100755 --- a/bin/mv_test.sh +++ b/bin/mv_test.sh @@ -160,27 +160,28 @@ fi CLASSPATH=${MV_HOME}/lib/xercesImpl.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/xml-apis.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/juli-6.0.53.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-api-1.7.5.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-log4j12-1.7.5.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-api-2.0.17.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/slf4j-log4j12-2.0.17.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/guava-33.4.8.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/junit-4.11.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/hamcrest-core-1.3.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-io-2.20.0.jar CLASSPATH=$CLASSPATH:${MV_HOME}/lib/mockito-all-1.9.5.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/servlet-api-4.0.1.jar -CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload-1.6.0.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/jakarta.servlet-api-5.0.0.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload2-core-2.0.0-M4.jar +CLASSPATH=$CLASSPATH:${MV_HOME}/lib/commons-fileupload2-jakarta-servlet5-2.0.0-M4.jar CLASSPATH=$CLASSPATH:${MV_HOME}/dist/lib/metviewer_all.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/j2html-1.4.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-core-2.20.0.jar CLASSPATH=$CLASSPATH:$MV_HOME/lib/jackson-databind-2.20.0.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.11.jar - -CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-2.7.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-8.5.61.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.17.1.jar -CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.0.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/commons-lang3-3.19.0.jar + +CLASSPATH=$CLASSPATH:$MV_HOME/lib/mariadb-java-client-3.5.6.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/tomcat-jdbc-10.1.46.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-api-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-core-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/log4j-iostreams-2.25.2.jar +CLASSPATH=$CLASSPATH:$MV_HOME/lib/snakeyaml-2.5.jar diff --git a/build.xml b/build.xml index 416007d6..47fa13f6 100644 --- a/build.xml +++ b/build.xml @@ -116,136 +116,136 @@ - - - - - + + + + + - - + + + - - - + + - + - - - - - + + + + + - - + + + - - - - + + + - - - - - + + + + + - - + + + - - - + + - + - - - - - + + + + + - - + + + - - - - + + + - - - - - + + + + + - - + + + - - - + + - + - - - - - + + + + + - - + + + - - - - + + + @@ -418,7 +418,7 @@ - + @@ -476,7 +476,7 @@ - + diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 28b10782..d5c2c75c 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base-metviewer -ARG MET_BASE_TAG=3.4-latest +ARG MET_BASE_TAG=3.5-latest FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} LABEL org.opencontainers.image.authors="John Halley Gotway " diff --git a/java/edu/ucar/metviewer/MVServlet.java b/java/edu/ucar/metviewer/MVServlet.java index 3324087c..2d87b784 100644 --- a/java/edu/ucar/metviewer/MVServlet.java +++ b/java/edu/ucar/metviewer/MVServlet.java @@ -1,12 +1,6 @@ package edu.ucar.metviewer; -import javax.servlet.ServletConfig; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; + import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -30,10 +24,18 @@ import edu.ucar.metviewer.db.AppDatabaseManager; import edu.ucar.metviewer.db.DatabaseManager; import edu.ucar.metviewer.jobManager.*; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.FileUploadException; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; -import org.apache.commons.fileupload.servlet.ServletFileUpload; +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.apache.commons.fileupload2.core.DiskFileItemFactory; +import org.apache.commons.fileupload2.core.FileItem; +import org.apache.commons.fileupload2.core.FileUploadException; +import org.apache.commons.fileupload2.jakarta.servlet5.JakartaServletFileUpload; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; @@ -1065,10 +1067,20 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) { // if the request is a file upload, build the request from the file XML - if (ServletFileUpload.isMultipartContent(request)) { + if (JakartaServletFileUpload.isMultipartContent(request)) { // set up the upload handler and parse the request - ServletFileUpload uploadHandler = new ServletFileUpload(new DiskFileItemFactory()); - List items = uploadHandler.parseRequest(request); + final DiskFileItemFactory fileItemfactory = DiskFileItemFactory.builder().get(); + final JakartaServletFileUpload fileUpload = new JakartaServletFileUpload(fileItemfactory); + List items; + try { + items = fileUpload.parseRequest(request); + } catch (FileUploadException e) { + items= new ArrayList<>(); + //throw new ServletException(e); + logger.error(e.getMessage()); + } + + // find the upload file in the request and read its contents StringBuilder uploadXml = new StringBuilder(); String error = null; @@ -1427,7 +1439,7 @@ else if (nodeCall.tag.equalsIgnoreCase("xml_upload")) { } } - } catch (ParserConfigurationException | FileUploadException | IOException | SAXException + } catch (ParserConfigurationException | IOException | SAXException | ServletException e) { errorStream.print("doPost() - caught " + e.getClass() + ": " + e.getMessage()); logger.info(INFO_MARKER, "doPost() - caught " + e.getClass() + ": " + e.getMessage()); diff --git a/java/edu/ucar/metviewer/MVUtil.java b/java/edu/ucar/metviewer/MVUtil.java index cf4cc9e1..84987e39 100644 --- a/java/edu/ucar/metviewer/MVUtil.java +++ b/java/edu/ucar/metviewer/MVUtil.java @@ -1,7 +1,6 @@ package edu.ucar.metviewer; import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.MarkerManager; import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.io.IoBuilder; import org.w3c.dom.Document; @@ -34,2915 +33,2953 @@ public class MVUtil { - private static final org.apache.logging.log4j.Logger logger = LogManager.getLogger("MVUtil"); - - - public static final Pattern thresh = Pattern.compile("([<>=!]{1,2})(\\d*(?:\\.\\d+)?)"); - public static final Pattern lev = Pattern.compile("(\\w)(\\d+)(?:-(\\d+))?"); - public static final Pattern interpPnts = Pattern.compile("\\d+?"); - public static final Pattern modeSingle = Pattern.compile("\\s+h\\.([^,]+),"); - public static final Pattern prob = Pattern.compile("PROB\\(([\\w\\d]+)([<>=]+)([^\\)]+)\\)"); - private static final Pattern plotTmpl = Pattern.compile("\\{((\\w+)(?:\\?[^}]*)?)\\}"); - private static final Pattern tag = Pattern.compile("([\\w\\d]+)(?:\\s*\\?(.*))?"); - - - public static final Pattern patModeSingleObjectId = Pattern.compile("^(C?[FO]\\d{3})$"); - public static final Pattern patModePairObjectId = Pattern.compile("^(C?F\\d{3})_(C?O\\d{3})$"); - public static final Pattern patDateTime = Pattern.compile("(\\d{4})-(\\d{2})-(\\d{2}) (\\d{2}):(\\d{2}):(\\d{2})"); - - public static final DateTimeFormatter APP_DATE_FORMATTER - = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - public final static String LINE_SEPARATOR = System.getProperty("line.separator"); - /** - * Parse the input mode statistic, which is assumed to have the form SSSS_FFF, where SSSS is the - * name of a mode statistic with arbitrary length and FFF is a three character flag indicator - * string. - */ - public static final Pattern _patModeStat = Pattern.compile("([^_]+)(?:_\\w{3})?_(\\w{2,3})"); - public static final String CTC = "ctc"; //Contingency Table Statistics - public static final String MCTC = "mctc"; //Multi-category Contingency Table Count - public static final String NBRCTC = "nbrctc"; //Contingency Table Statistics - public static final String SL1L2 = "sl1l2"; //Scalar partial sums - public static final String GRAD = "grad"; //Gradient partial sums - public static final String SAL1L2 = "sal1l2"; // Scalar anomaly partial sums - public static final String SSVAR = "ssvar"; // Spread/Skill Variance - public static final String PCT = "pct"; //Contingency Table Counts for Probabilistic forecasts - public static final String NBRCNT = "nbrcnt"; // Neighborhood Continuous Statistics - public static final String VL1L2 = "vl1l2"; // Vector Partial Sum - public static final String VAL1L2 = "val1l2"; // Vector Anomaly Partial Sum - public static final String VCNT = "vcnt"; // Vector Continuous Statistics - public static final String ECNT = "ecnt"; // Ensemble Continuous Statistics - public static final String RPS = "rps"; // Ranked Probability Score Statistics - public static final Map statsEnscnt = new HashMap<>(); - public static final Map statsMpr = new HashMap<>(); - public static final Map statsDmap = new HashMap<>(); - public static final Map statsOrank = new HashMap<>(); - public static final Map statsCnt = new HashMap<>(); - public static final Map statsVcnt = new HashMap<>(); - public static final Map statsEcnt = new HashMap<>(); - public static final Map statsSsvar = new HashMap<>(); - public static final Map statsCts = new HashMap<>(); - public static final Map statsNbrcts = new HashMap<>(); - public static final Map statsNbrcnt = new HashMap<>(); - public static final Map statsPstd = new HashMap<>(); - public static final Map statsMcts = new HashMap<>(); - public static final Map statsPhist = new HashMap<>(); - public static final Map statsRhist = new HashMap<>(); - public static final Map statsVl1l2 = new HashMap<>(); - public static final Map statsVal1l2 = new HashMap<>(); - public static final Map statsPerc = new HashMap<>(); - public static final Map modeSingleStatField = new HashMap<>(); - public static final Map mtd3dSingleStatField = new HashMap<>(); - public static final Map mtd2dStatField = new HashMap<>(); - public static final Map statsRps = new HashMap<>(); - public static final Map statsCtc = new HashMap<>(); - public static final Map statsNbrctc = new HashMap<>(); - public static final Map statsPct = new HashMap<>(); - public static final Map statsSl1l2 = new HashMap<>(); - public static final Map statsSal1l2 = new HashMap<>(); - public static final Map statsGrad = new HashMap<>(); - public static final Map statsMctc = new HashMap<>(); - public static final Map statsSsidx = new HashMap<>(); - public static final Map statsSeeps = new HashMap<>(); - - - public static final int MAX_STR_LEN = 500; - - - public static final List modeRatioField = new ArrayList<>(); - public static final List mtdRatioField = new ArrayList<>(); - - - public static final Map modePairStatField = new HashMap<>(); - public static final Map mtd3dPairStatField = new HashMap<>(); - public static final List calcStatCTC = new ArrayList<>(); - public static final String DB_DATE_MS = "yyyy-MM-dd HH:mm:ss.S"; - public static final SimpleDateFormat PLOT_FORMAT = new SimpleDateFormat("yyyyMMddHH"); - - - public static final Pattern patRTmpl = Pattern.compile("#<(\\w+)>#"); - - public static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("0.000"); - public static final String PYTHON = "Python"; - public static final String RSCRIPT = "Rscript"; - - /* - * variable length group data indices for lines with an arbitrary number of fields - * - index of field containing number of sets - * - index of first repeating field(s) - * - number of fields in each repeating set - */ - public static final Map lengthGroupIndices = new HashMap<>(); - public static final Map alphaLineTypes = new HashMap<>(); - - public static final Map covThreshLineTypes = new HashMap<>(); - - public static final PrintStream errorStream = IoBuilder.forLogger(MVUtil.class) - .setLevel(org.apache.logging.log4j.Level.INFO) - .buildPrintStream(); - public static final String DEFAULT_DATABASE_GROUP = "NO GROUP"; - - public static final String SEPARATOR = ","; - public static final String GROUP_SEPARATOR = ":"; - - public static final String[] lineTypes = new String[]{ - "fho", - "ctc", - "cts", - "mctc", - "mcts", - "cnt", - "sl1l2", - "sal1l2", - "vl1l2", - "val1l2", - "pct", - "pstd", - "pjc", - "prc", - "eclv", - "mpr", - "nbrctc", - "nbrcts", - "nbrcnt", - "isc", - "rhist", - "phist", - "orank", - "ssvar", - "grad", - "vcnt", - "relp", - "ecnt", - "dmap", - "rps", - "seeps" - }; - - static { - - modeRatioField.add("RATIO_FSA_ASA"); - modeRatioField.add("RATIO_OSA_ASA"); - modeRatioField.add("RATIO_ASM_ASA"); - modeRatioField.add("RATIO_ASU_ASA"); - modeRatioField.add("RATIO_FSM_FSA"); - modeRatioField.add("RATIO_FSU_FSA"); - modeRatioField.add("RATIO_OSM_OSA"); - modeRatioField.add("RATIO_OSU_OSA"); - modeRatioField.add("RATIO_FSM_ASM"); - modeRatioField.add("RATIO_OSM_ASM"); - modeRatioField.add("RATIO_FSU_ASU"); - modeRatioField.add("RATIO_OSU_ASU"); - modeRatioField.add("RATIO_FSA_AAA"); - modeRatioField.add("RATIO_OSA_AAA"); - modeRatioField.add("RATIO_FSA_FAA"); - modeRatioField.add("RATIO_FCA_FAA"); - modeRatioField.add("RATIO_OSA_OAA"); - modeRatioField.add("RATIO_OCA_OAA"); - modeRatioField.add("RATIO_FCA_ACA"); - modeRatioField.add("RATIO_OCA_ACA"); - modeRatioField.add("RATIO_FSA_OSA"); - modeRatioField.add("RATIO_OSA_FSA"); - modeRatioField.add("RATIO_ACA_ASA"); - modeRatioField.add("RATIO_ASA_ACA"); - modeRatioField.add("RATIO_FCA_FSA"); - modeRatioField.add("RATIO_FSA_FCA"); - modeRatioField.add("RATIO_OCA_OSA"); - modeRatioField.add("RATIO_OSA_OCA"); - - modeRatioField.add("OBJHITS"); - modeRatioField.add("OBJMISSES"); - modeRatioField.add("OBJFAS"); - modeRatioField.add("OBJCSI"); - modeRatioField.add("OBJPODY"); - modeRatioField.add("OBJFAR"); - - modeRatioField.add("AREARAT_FSA_ASA"); - modeRatioField.add("AREARAT_OSA_ASA"); - modeRatioField.add("AREARAT_ASM_ASA"); - modeRatioField.add("AREARAT_ASU_ASA"); - modeRatioField.add("AREARAT_FSM_FSA"); - modeRatioField.add("AREARAT_FSU_FSA"); - modeRatioField.add("AREARAT_OSM_OSA"); - modeRatioField.add("AREARAT_OSU_OSA"); - modeRatioField.add("AREARAT_FSM_ASM"); - modeRatioField.add("AREARAT_OSM_ASM"); - modeRatioField.add("AREARAT_FSU_ASU"); - modeRatioField.add("AREARAT_OSU_ASU"); - modeRatioField.add("AREARAT_FSA_AAA"); - modeRatioField.add("AREARAT_OSA_AAA"); - modeRatioField.add("AREARAT_FSA_FAA"); - modeRatioField.add("AREARAT_FCA_FAA"); - modeRatioField.add("AREARAT_OSA_OAA"); - modeRatioField.add("AREARAT_OCA_OAA"); - modeRatioField.add("AREARAT_FCA_ACA"); - modeRatioField.add("AREARAT_OCA_ACA"); - modeRatioField.add("AREARAT_FSA_OSA"); - modeRatioField.add("AREARAT_OSA_FSA"); - modeRatioField.add("AREARAT_ACA_ASA"); - modeRatioField.add("AREARAT_ASA_ACA"); - modeRatioField.add("AREARAT_FCA_FSA"); - modeRatioField.add("AREARAT_FSA_FCA"); - modeRatioField.add("AREARAT_OCA_OSA"); - modeRatioField.add("AREARAT_OSA_OCA"); - - modeRatioField.add("OBJAHITS"); - modeRatioField.add("OBJAMISSES"); - modeRatioField.add("OBJAFAS"); - modeRatioField.add("OBJACSI"); - modeRatioField.add("OBJAPODY"); - modeRatioField.add("OBJAFAR"); - } - - - static { - - mtdRatioField.add("2d_RATIO_FSA_ASA"); - mtdRatioField.add("2d_RATIO_OSA_ASA"); - mtdRatioField.add("2d_RATIO_ASM_ASA"); - mtdRatioField.add("2d_RATIO_ASU_ASA"); - mtdRatioField.add("2d_RATIO_FSM_FSA"); - mtdRatioField.add("2d_RATIO_FSU_FSA"); - mtdRatioField.add("2d_RATIO_OSM_OSA"); - mtdRatioField.add("2d_RATIO_OSU_OSA"); - mtdRatioField.add("2d_RATIO_FSM_ASM"); - mtdRatioField.add("2d_RATIO_OSM_ASM"); - mtdRatioField.add("2d_RATIO_FSU_ASU"); - mtdRatioField.add("2d_RATIO_OSU_ASU"); - mtdRatioField.add("2d_RATIO_FSA_AAA"); - mtdRatioField.add("2d_RATIO_OSA_AAA"); - mtdRatioField.add("2d_RATIO_FSA_FAA"); - mtdRatioField.add("2d_RATIO_FCA_FAA"); - mtdRatioField.add("2d_RATIO_OSA_OAA"); - mtdRatioField.add("2d_RATIO_OCA_OAA"); - mtdRatioField.add("2d_RATIO_FCA_ACA"); - mtdRatioField.add("2d_RATIO_OCA_ACA"); - mtdRatioField.add("2d_RATIO_FSA_OSA"); - mtdRatioField.add("2d_RATIO_OSA_FSA"); - mtdRatioField.add("2d_RATIO_ACA_ASA"); - mtdRatioField.add("2d_RATIO_ASA_ACA"); - mtdRatioField.add("2d_RATIO_FCA_FSA"); - mtdRatioField.add("2d_RATIO_FSA_FCA"); - mtdRatioField.add("2d_RATIO_OCA_OSA"); - mtdRatioField.add("2d_RATIO_OSA_OCA"); - - - mtdRatioField.add("3d_RATIO_FSA_ASA"); - mtdRatioField.add("3d_RATIO_OSA_ASA"); - mtdRatioField.add("3d_RATIO_ASM_ASA"); - mtdRatioField.add("3d_RATIO_ASU_ASA"); - mtdRatioField.add("3d_RATIO_FSM_FSA"); - mtdRatioField.add("3d_RATIO_FSU_FSA"); - mtdRatioField.add("3d_RATIO_OSM_OSA"); - mtdRatioField.add("3d_RATIO_OSU_OSA"); - mtdRatioField.add("3d_RATIO_FSM_ASM"); - mtdRatioField.add("3d_RATIO_OSM_ASM"); - mtdRatioField.add("3d_RATIO_FSU_ASU"); - mtdRatioField.add("3d_RATIO_OSU_ASU"); - mtdRatioField.add("3d_RATIO_FSA_AAA"); - mtdRatioField.add("3d_RATIO_OSA_AAA"); - mtdRatioField.add("3d_RATIO_FSA_FAA"); - mtdRatioField.add("3d_RATIO_FCA_FAA"); - mtdRatioField.add("3d_RATIO_OSA_OAA"); - mtdRatioField.add("3d_RATIO_OCA_OAA"); - mtdRatioField.add("3d_RATIO_FCA_ACA"); - mtdRatioField.add("3d_RATIO_OCA_ACA"); - mtdRatioField.add("3d_RATIO_FSA_OSA"); - mtdRatioField.add("3d_RATIO_OSA_FSA"); - mtdRatioField.add("3d_RATIO_ACA_ASA"); - mtdRatioField.add("3d_RATIO_ASA_ACA"); - mtdRatioField.add("3d_RATIO_FCA_FSA"); - mtdRatioField.add("3d_RATIO_FSA_FCA"); - mtdRatioField.add("3d_RATIO_OCA_OSA"); - mtdRatioField.add("3d_RATIO_OSA_OCA"); - - mtdRatioField.add("2d_OBJHITS"); - mtdRatioField.add("2d_OBJMISSES"); - mtdRatioField.add("2d_OBJFAS"); - mtdRatioField.add("2d_OBJCSI"); - mtdRatioField.add("2d_OBJPODY"); - mtdRatioField.add("2d_OBJFAR"); - mtdRatioField.add("3d_OBJHITS"); - mtdRatioField.add("3d_OBJMISSES"); - mtdRatioField.add("3d_OBJFAS"); - mtdRatioField.add("3d_OBJCSI"); - mtdRatioField.add("3d_OBJPODY"); - mtdRatioField.add("3d_OBJFAR"); - - mtdRatioField.add("3d_VOLRAT_FSA_ASA"); - mtdRatioField.add("3d_VOLRAT_OSA_ASA"); - mtdRatioField.add("3d_VOLRAT_ASM_ASA"); - mtdRatioField.add("3d_VOLRAT_ASU_ASA"); - mtdRatioField.add("3d_VOLRAT_FSM_FSA"); - mtdRatioField.add("3d_VOLRAT_FSU_FSA"); - mtdRatioField.add("3d_VOLRAT_OSM_OSA"); - mtdRatioField.add("3d_VOLRAT_OSU_OSA"); - mtdRatioField.add("3d_VOLRAT_FSM_ASM"); - mtdRatioField.add("3d_VOLRAT_OSM_ASM"); - mtdRatioField.add("3d_VOLRAT_FSU_ASU"); - mtdRatioField.add("3d_VOLRAT_OSU_ASU"); - mtdRatioField.add("3d_VOLRAT_FSA_AAA"); - mtdRatioField.add("3d_VOLRAT_OSA_AAA"); - mtdRatioField.add("3d_VOLRAT_FSA_FAA"); - mtdRatioField.add("3d_VOLRAT_FCA_FAA"); - mtdRatioField.add("3d_VOLRAT_OSA_OAA"); - mtdRatioField.add("3d_VOLRAT_OCA_OAA"); - mtdRatioField.add("3d_VOLRAT_FCA_ACA"); - mtdRatioField.add("3d_VOLRAT_OCA_ACA"); - mtdRatioField.add("3d_VOLRAT_FSA_OSA"); - mtdRatioField.add("3d_VOLRAT_OSA_FSA"); - mtdRatioField.add("3d_VOLRAT_ACA_ASA"); - mtdRatioField.add("3d_VOLRAT_ASA_ACA"); - mtdRatioField.add("3d_VOLRAT_FCA_FSA"); - mtdRatioField.add("3d_VOLRAT_FSA_FCA"); - mtdRatioField.add("3d_VOLRAT_OCA_OSA"); - mtdRatioField.add("3d_VOLRAT_OSA_OCA"); - - mtdRatioField.add("2d_AREARAT_FSA_ASA"); - mtdRatioField.add("2d_AREARAT_OSA_ASA"); - mtdRatioField.add("2d_AREARAT_ASM_ASA"); - mtdRatioField.add("2d_AREARAT_ASU_ASA"); - mtdRatioField.add("2d_AREARAT_FSM_FSA"); - mtdRatioField.add("2d_AREARAT_FSU_FSA"); - mtdRatioField.add("2d_AREARAT_OSM_OSA"); - mtdRatioField.add("2d_AREARAT_OSU_OSA"); - mtdRatioField.add("2d_AREARAT_FSM_ASM"); - mtdRatioField.add("2d_AREARAT_OSM_ASM"); - mtdRatioField.add("2d_AREARAT_FSU_ASU"); - mtdRatioField.add("2d_AREARAT_OSU_ASU"); - mtdRatioField.add("2d_AREARAT_FSA_AAA"); - mtdRatioField.add("2d_AREARAT_OSA_AAA"); - mtdRatioField.add("2d_AREARAT_FSA_FAA"); - mtdRatioField.add("2d_AREARAT_FCA_FAA"); - mtdRatioField.add("2d_AREARAT_OSA_OAA"); - mtdRatioField.add("2d_AREARAT_OCA_OAA"); - mtdRatioField.add("2d_AREARAT_FCA_ACA"); - mtdRatioField.add("2d_AREARAT_OCA_ACA"); - mtdRatioField.add("2d_AREARAT_FSA_OSA"); - mtdRatioField.add("2d_AREARAT_OSA_FSA"); - mtdRatioField.add("2d_AREARAT_ACA_ASA"); - mtdRatioField.add("2d_AREARAT_ASA_ACA"); - mtdRatioField.add("2d_AREARAT_FCA_FSA"); - mtdRatioField.add("2d_AREARAT_FSA_FCA"); - mtdRatioField.add("2d_AREARAT_OCA_OSA"); - mtdRatioField.add("2d_AREARAT_OSA_OCA"); - - mtdRatioField.add("3d_OBJVHITS"); - mtdRatioField.add("3d_OBJVMISSES"); - mtdRatioField.add("3d_OBJVFAS"); - mtdRatioField.add("3d_OBJVCSI"); - mtdRatioField.add("3d_OBJVPODY"); - mtdRatioField.add("3d_OBJVFAR"); - mtdRatioField.add("3d_OBJVHITS"); - mtdRatioField.add("2d_OBJAMISSES"); - mtdRatioField.add("2d_OBJAFAS"); - mtdRatioField.add("2d_OBJACSI"); - mtdRatioField.add("2d_OBJAPODY"); - mtdRatioField.add("2d_OBJAFAR"); - } - - - static { - covThreshLineTypes.put("NBRCTC", Boolean.TRUE); - covThreshLineTypes.put("NBRCTS", Boolean.TRUE); - covThreshLineTypes.put("PCT", Boolean.TRUE); - covThreshLineTypes.put("PSTD", Boolean.TRUE); - covThreshLineTypes.put("PJC", Boolean.TRUE); - covThreshLineTypes.put("PRC", Boolean.TRUE); - } - - static { - alphaLineTypes.put("CTS", Boolean.TRUE); - alphaLineTypes.put("NBRCTS", Boolean.TRUE); - alphaLineTypes.put("NBRCNT", Boolean.TRUE); - alphaLineTypes.put("MCTS", Boolean.TRUE); - alphaLineTypes.put("SSVAR", Boolean.TRUE); - alphaLineTypes.put("VCNT", Boolean.TRUE); - alphaLineTypes.put("DMAP", Boolean.TRUE); - alphaLineTypes.put("RPS", Boolean.TRUE); - - alphaLineTypes.put("CNT", Boolean.FALSE); - alphaLineTypes.put("PSTD", Boolean.FALSE); - } - - - static { - lengthGroupIndices.put("PCT", new int[]{23, 24, 3}); - lengthGroupIndices.put("PSTD", new int[]{23, 33, 1}); - lengthGroupIndices.put("PJC", new int[]{23, 24, 7}); - lengthGroupIndices.put("PRC", new int[]{23, 24, 3}); - lengthGroupIndices.put("MCTC", new int[]{23, 24, 1}); - lengthGroupIndices.put("RHIST", new int[]{23, 23, 1}); - lengthGroupIndices.put("PHIST", new int[]{24, 25, 1}); - lengthGroupIndices.put("RELP", new int[]{23, 24, 1}); - lengthGroupIndices.put("ORANK", new int[]{33, 34, 1}); - lengthGroupIndices.put("ECLV", new int[]{25, 26, 2}); - } - - static { - statsEnscnt.put("ENS_RPSF", new String[]{}); - statsEnscnt.put("ENS_RPSCL", new String[]{}); - statsEnscnt.put("ENS_RPSS", new String[]{}); - statsEnscnt.put("ENS_CRPSF", new String[]{}); - statsEnscnt.put("ENS_CRPSCL", new String[]{}); - statsEnscnt.put("ENS_CRPSS", new String[]{}); - } - - static { - statsMpr.put("MPR_FCST", new String[]{""}); - statsMpr.put("MPR_OBS", new String[]{""}); - statsMpr.put("MPR_CLIMO", new String[]{""}); - } - - static { - statsDmap.put("DMAP_FBIAS", new String[]{""}); - statsDmap.put("DMAP_BADDELEY", new String[]{""}); - statsDmap.put("DMAP_HAUSDORFF", new String[]{""}); - statsDmap.put("DMAP_MED_FO", new String[]{""}); - statsDmap.put("DMAP_MED_OF", new String[]{""}); - statsDmap.put("DMAP_MED_MIN", new String[]{""}); - statsDmap.put("DMAP_MED_MAX", new String[]{""}); - statsDmap.put("DMAP_MED_MEAN", new String[]{""}); - statsDmap.put("DMAP_FOM_FO", new String[]{""}); - statsDmap.put("DMAP_FOM_OF", new String[]{""}); - statsDmap.put("DMAP_FOM_MIN", new String[]{""}); - statsDmap.put("DMAP_FOM_MAX", new String[]{""}); - statsDmap.put("DMAP_FOM_MEAN", new String[]{""}); - statsDmap.put("DMAP_ZHU_FO", new String[]{""}); - statsDmap.put("DMAP_ZHU_OF", new String[]{""}); - statsDmap.put("DMAP_ZHU_MIN", new String[]{""}); - statsDmap.put("DMAP_ZHU_MAX", new String[]{""}); - statsDmap.put("DMAP_ZHU_MEAN", new String[]{""}); - statsDmap.put("DMAP_G", new String[]{""}); - statsDmap.put("DMAP_GBETA", new String[]{""}); - } - - static { - statsRps.put("RPS_REL", new String[]{""}); - statsRps.put("RPS_RES", new String[]{""}); - statsRps.put("RPS_UNC", new String[]{""}); - statsRps.put("RPS", new String[]{RPS}); - statsRps.put("RPSS", new String[]{RPS}); - statsRps.put("RPSS_SMPL", new String[]{""}); - statsRps.put("RPS_COMP", new String[]{RPS}); - statsRps.put("RPS_TOTAL", new String[]{RPS}); - } - - static { - statsCtc.put("CTC_FY_OY", new String[]{CTC}); - statsCtc.put("CTC_FY_ON", new String[]{CTC}); - statsCtc.put("CTC_FN_OY", new String[]{CTC}); - statsCtc.put("CTC_FN_ON", new String[]{CTC}); - statsCtc.put("CTC_OY", new String[]{CTC}); - statsCtc.put("CTC_ON", new String[]{CTC}); - statsCtc.put("CTC_FY", new String[]{CTC}); - statsCtc.put("CTC_FN", new String[]{CTC}); - statsCtc.put("CTC_TOTAL", new String[]{CTC}); - } - - static { - statsOrank.put("ORANK_PIT", new String[]{""}); - statsOrank.put("ORANK_OBS_RANK", new String[]{""}); - statsOrank.put("ORANK_ENS_MEAN", new String[]{""}); - statsOrank.put("ORANK_SPREAD", new String[]{""}); - statsOrank.put("ORANK_CLIMO_MEAN", new String[]{""}); - statsOrank.put("ORANK_ENS_MEAN_OERR", new String[]{""}); - statsOrank.put("ORANK_SPREAD_OERR", new String[]{""}); - statsOrank.put("ORANK_SPREAD_PLUS_OERR", new String[]{""}); - statsOrank.put("ORANK_CLIMO_STDEV", new String[]{""}); - } - - static { - statsCnt.put("FBAR", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("FSTDEV", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("OBAR", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("OSTDEV", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("PR_CORR", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("SP_CORR", new String[]{}); - statsCnt.put("KT_CORR", new String[]{}); - statsCnt.put("ME", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("ESTDEV", new String[]{"nc", "bc", SL1L2}); - statsCnt.put("MBIAS", new String[]{"bc", SL1L2}); - statsCnt.put("MAE", new String[]{"bc", SL1L2}); - statsCnt.put("MSE", new String[]{"bc", SL1L2}); - statsCnt.put("BCMSE", new String[]{"bc", SL1L2}); - statsCnt.put("BCRMSE", new String[]{"bc", SL1L2}); - statsCnt.put("RMSE", new String[]{"bc", SL1L2}); - statsCnt.put("FGBAR", new String[]{"bc", GRAD}); - statsCnt.put("OGBAR", new String[]{"bc", GRAD}); - statsCnt.put("MGBAR", new String[]{"bc", GRAD}); - statsCnt.put("EGBAR", new String[]{"bc", GRAD}); - statsCnt.put("S1", new String[]{"bc", GRAD}); - statsCnt.put("S1_OG", new String[]{"bc", GRAD}); - statsCnt.put("FGOG_RATIO", new String[]{"bc", GRAD}); - statsCnt.put("E10", new String[]{"bc"}); - statsCnt.put("E25", new String[]{"bc"}); - statsCnt.put("E50", new String[]{"bc"}); - statsCnt.put("E75", new String[]{"bc"}); - statsCnt.put("E90", new String[]{"bc"}); - statsCnt.put("IQR", new String[]{"bc"}); - statsCnt.put("MAD", new String[]{"bc"}); - //statsCnt.put("PAC", new String[]{"bc"}); - statsCnt.put("ANOM_CORR", new String[]{"bc", SAL1L2}); - statsCnt.put("ANOM_CORR_UNCNTR", new String[]{"bc", SAL1L2}); - statsCnt.put("ME2", new String[]{"bc", SL1L2}); - statsCnt.put("MSESS", new String[]{"bc", SL1L2}); - statsCnt.put("RMSFA", new String[]{"bc", SAL1L2}); - statsCnt.put("RMSOA", new String[]{"bc", SAL1L2}); - statsCnt.put("SI", new String[]{"bc", SL1L2}); - } - - static { - statsSl1l2.put("SL1L2_TOTAL", new String[]{SL1L2}); - } - - static { - statsSal1l2.put("SAL1L2_TOTAL", new String[]{SAL1L2}); - } - - static { - statsGrad.put("GRAD_TOTAL", new String[]{GRAD}); - } - - static { - statsVcnt.put("VCNT_FBAR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_OBAR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_FS_RMS", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_OS_RMS", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_MSVE", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_RMSVE", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_FSTDEV", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_OSTDEV", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_FDIR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_ODIR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_FBAR_SPEED", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_OBAR_SPEED", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_VDIFF_SPEED", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_VDIFF_DIR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_SPEED_ERR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_SPEED_ABSERR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_ERR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_ABSERR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_ANOM_CORR", new String[]{"nc","bc", VCNT}); - statsVcnt.put("VCNT_ANOM_CORR_UNCNTR", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_ME", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_MAE", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_MSE", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_DIR_RMSE", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_TOTAL", new String[]{"bc", VCNT}); - statsVcnt.put("VCNT_TOTAL_DIR", new String[]{"bc", VCNT}); - - - } - - static { - statsEcnt.put("ECNT_CRPS", new String[]{ ECNT}); - statsEcnt.put("ECNT_CRPSS", new String[]{ECNT}); - statsEcnt.put("ECNT_IGN", new String[]{ECNT}); - statsEcnt.put("ECNT_ME", new String[]{ECNT}); - statsEcnt.put("ECNT_RMSE", new String[]{ECNT}); - statsEcnt.put("ECNT_SPREAD", new String[]{ECNT}); - statsEcnt.put("ECNT_ME_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_RMSE_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_SPREAD_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_SPREAD_PLUS_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_CRPSCL", new String[]{ECNT}); - statsEcnt.put("ECNT_CRPS_EMP", new String[]{ECNT}); - statsEcnt.put("ECNT_CRPSCL_EMP", new String[]{ECNT}); - statsEcnt.put("ECNT_CRPSS_EMP", new String[]{ECNT}); - statsEcnt.put("ECNT_CRPS_EMP_FAIR", new String[]{ECNT}); - statsEcnt.put("ECNT_SPREAD_MD", new String[]{ECNT}); - statsEcnt.put("ECNT_MAE", new String[]{ECNT}); - statsEcnt.put("ECNT_MAE_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_BIAS_RATIO", new String[]{ECNT}); - statsEcnt.put("ECNT_N_GE_OBS", new String[]{ECNT}); - statsEcnt.put("ECNT_ME_GE_OBS", new String[]{ECNT}); - statsEcnt.put("ECNT_N_LT_OBS", new String[]{ECNT}); - statsEcnt.put("ECNT_ME_LT_OBS", new String[]{ECNT}); - statsEcnt.put("ECNT_IGN_CONV_OERR", new String[]{ECNT}); - statsEcnt.put("ECNT_IGN_CORR_OERR", new String[]{ECNT}); - - statsEcnt.put("ECNT_TOTAL", new String[]{ECNT}); - } - - static { - statsSsvar.put("SSVAR_FBAR", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_FSTDEV", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_OBAR", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_OSTDEV", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_PR_CORR", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_ME", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_ESTDEV", new String[]{"nc", "bc", SSVAR}); - statsSsvar.put("SSVAR_MBIAS", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_MSE", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_BCMSE", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_BCRMSE", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_RMSE", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_ANOM_CORR", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_ME2", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_MSESS", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_Spread", new String[]{"bc", SSVAR}); - statsSsvar.put("SSVAR_TOTAL", new String[]{SSVAR}); - } - - - static { - statsCts.put("BASER", new String[]{"nc", "bc", CTC}); - statsCts.put("FMEAN", new String[]{"nc", "bc", CTC}); - statsCts.put("ACC", new String[]{"nc", "bc", CTC}); - statsCts.put("FBIAS", new String[]{"bc", CTC}); - statsCts.put("PODY", new String[]{"nc", "bc", CTC}); - statsCts.put("PODN", new String[]{"nc", "bc", CTC}); - statsCts.put("POFD", new String[]{"nc", "bc", CTC}); - statsCts.put("FAR", new String[]{"nc", "bc", CTC}); - statsCts.put("CSI", new String[]{"nc", "bc", CTC}); - statsCts.put("GSS", new String[]{"bc", CTC}); - statsCts.put("HK", new String[]{"nc", "bc", CTC}); - statsCts.put("HSS", new String[]{"bc", CTC}); - statsCts.put("ODDS", new String[]{"nc", "bc", CTC}); - statsCts.put("LODDS", new String[]{"nc", "bc", CTC}); - statsCts.put("ORSS", new String[]{"nc", "bc", CTC}); - statsCts.put("EDS", new String[]{"nc", "bc", CTC}); - statsCts.put("SEDS", new String[]{"nc", "bc", CTC}); - statsCts.put("EDI", new String[]{"nc", "bc", CTC}); - statsCts.put("SEDI", new String[]{"nc", "bc", CTC}); - statsCts.put("BAGSS", new String[]{"bc", CTC}); - statsCts.put("HSS_EC", new String[]{"bc", CTC}); - } - - static { - statsMctc.put("MCTS_HSS_EC", new String[]{"bc", MCTC}); - } - static { - statsSsidx.put("SS_INDEX", new String[]{}); - } - static { - statsSeeps.put("SEEPS_MEAN_FCST", new String[]{}); - statsSeeps.put("SEEPS_MEAN_OBS", new String[]{}); - statsSeeps.put("SEEPS", new String[]{}); - statsSeeps.put("SEEPS_S12", new String[]{}); - statsSeeps.put("SEEPS_S13", new String[]{}); - statsSeeps.put("SEEPS_S21", new String[]{}); - statsSeeps.put("SEEPS_S23", new String[]{}); - statsSeeps.put("SEEPS_S31", new String[]{}); - statsSeeps.put("SEEPS_S32", new String[]{}); - statsSeeps.put("SEEPS_PF1", new String[]{}); - statsSeeps.put("SEEPS_PF2", new String[]{}); - statsSeeps.put("SEEPS_PF3", new String[]{}); - statsSeeps.put("SEEPS_PV1", new String[]{}); - statsSeeps.put("SEEPS_PV2", new String[]{}); - statsSeeps.put("SEEPS_PV3", new String[]{}); - } - - - - static { - statsNbrcts.put("NBR_BASER", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_FMEAN", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_ACC", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_FBIAS", new String[]{"bc", NBRCTC}); - statsNbrcts.put("NBR_PODY", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_PODN", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_POFD", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_FAR", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_CSI", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_GSS", new String[]{"bc", NBRCTC}); - statsNbrcts.put("NBR_HK", new String[]{"nc", "bc", NBRCTC}); - statsNbrcts.put("NBR_HSS", new String[]{"bc", NBRCTC}); - statsNbrcts.put("NBR_ODDS", new String[]{"nc", "bc", NBRCTC}); - } - - static { - statsNbrctc.put("NBR_CTC_TOTAL", new String[]{NBRCTC}); - } - - static { - statsNbrcnt.put("NBR_FBS", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_FSS", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_AFSS", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_UFSS", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_F_RATE", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_O_RATE", new String[]{"bc", NBRCNT}); - statsNbrcnt.put("NBR_CNT_TOTAL", new String[]{NBRCNT}); - } - - static { - statsPstd.put("PSTD_BASER", new String[]{"nc", PCT}); - statsPstd.put("PSTD_RELIABILITY", new String[]{PCT}); - statsPstd.put("PSTD_RESOLUTION", new String[]{PCT}); - statsPstd.put("PSTD_UNCERTAINTY", new String[]{PCT}); - statsPstd.put("PSTD_ROC_AUC", new String[]{PCT}); - statsPstd.put("PSTD_BRIER", new String[]{"nc", PCT}); - statsPstd.put("PSTD_BRIERCL", new String[]{}); - statsPstd.put("PSTD_BSS", new String[]{}); - statsPstd.put("PSTD_BSS_SMPL", new String[]{PCT}); - } - - static { - statsPct.put("PCT_TOTAL", new String[]{PCT}); - } - - static { - statsMcts.put("MCTS_ACC", new String[]{"nc", "bc"}); - statsMcts.put("MCTS_HK", new String[]{"bc"}); - statsMcts.put("MCTS_HSS", new String[]{"bc"}); - statsMcts.put("MCTS_GER", new String[]{"bc"}); - statsMcts.put("MCTS_HSS_EC", new String[]{"bc"}); - //statsMcts.put("MCTS_TOTAL", new String[]{MCTC}); - } - - static { - statsRhist.put("RHIST_CRPS", new String[]{}); - statsRhist.put("RHIST_CRPSS", new String[]{}); - statsRhist.put("RHIST_IGN", new String[]{}); - statsRhist.put("RHIST_SPREAD", new String[]{}); - } - - static { - statsVal1l2.put("VAL1L2_ANOM_CORR", new String[]{VAL1L2}); - statsVal1l2.put("VAL1L2_DIRA_ME", new String[]{VAL1L2}); - statsVal1l2.put("VAL1L2_DIRA_MAE", new String[]{VAL1L2}); - statsVal1l2.put("VAL1L2_DIRA_MSE", new String[]{VAL1L2}); - statsVal1l2.put("VAL1L2_TOTAL", new String[]{VAL1L2}); - } - - static { - statsVl1l2.put("VL1L2_UFBAR", new String[]{}); - statsVl1l2.put("VL1L2_VFBAR", new String[]{}); - statsVl1l2.put("VL1L2_UOBAR", new String[]{}); - statsVl1l2.put("VL1L2_VOBAR", new String[]{}); - statsVl1l2.put("VL1L2_BIAS", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_FVAR", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_OVAR", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_SPEED_ERR", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_RMSVE", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_SPEED_DIFF", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_DIR_ME", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_DIR_MAE", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_DIR_MSE", new String[]{VL1L2}); - statsVl1l2.put("VL1L2_TOTAL", new String[]{VL1L2}); - } - - public static final String COUNT = "COUNT(*)"; - - static { - modeSingleStatField.put("ACOV", "SUM(area)"); - modeSingleStatField.put("CNT", COUNT); - modeSingleStatField.put("CNTSUM", COUNT); - modeSingleStatField.put("CENTX", "centroid_x"); - modeSingleStatField.put("CENTY", "centroid_y"); - modeSingleStatField.put("CENTLAT", "centroid_lat"); - modeSingleStatField.put("CENTLON", "centroid_lon"); - modeSingleStatField.put("AXAVG", "axis_avg"); - modeSingleStatField.put("LEN", "length"); - modeSingleStatField.put("WID", "width"); - modeSingleStatField - .put("ASPECT", "IF((length/width) < (width/length), length/width, width/length)"); - modeSingleStatField.put("AREA", "area"); - modeSingleStatField.put("AREATHR", "area_thresh"); - modeSingleStatField.put("CURV", "curvature"); - modeSingleStatField.put("CURVX", "curvature_x"); - modeSingleStatField.put("CURVY", "curvature_y"); - modeSingleStatField.put("CPLX", "complexity"); - modeSingleStatField.put("INT10", "intensity_10"); - modeSingleStatField.put("INT25", "intensity_25"); - modeSingleStatField.put("INT50", "intensity_50"); - modeSingleStatField.put("INT75", "intensity_75"); - modeSingleStatField.put("INT90", "intensity_90"); - modeSingleStatField.put("INTN", "intensity_nn"); - modeSingleStatField.put("INTSUM", "intensity_sum"); - - } - - - static { - modePairStatField.put("CENTDIST", "centroid_dist"); - modePairStatField.put("BOUNDDIST", "boundary_dist"); - modePairStatField.put("HULLDIST", "convex_hull_dist"); - modePairStatField.put("ANGLEDIFF", "angle_diff"); - modePairStatField.put("AREARATIO", "area_ratio"); - modePairStatField.put("INTAREA", "intersection_area"); - modePairStatField.put("UNIONAREA", "union_area"); - modePairStatField.put("SYMDIFF", "symmetric_diff"); - modePairStatField.put("INTOVERAREA", "intersection_over_area"); - modePairStatField.put("CMPLXRATIO", "complexity_ratio"); - modePairStatField.put("PERCINTRATIO", "percentile_intensity_ratio"); - modePairStatField.put("INT", "interest"); - modePairStatField.put("MAXINT", "MAX(interest)"); - modePairStatField.put("MAXINTF", "MAX(interest)"); - modePairStatField.put("MAXINTO", "MAX(interest)"); - modePairStatField.put("ASPECTDIFF", "aspect_diff"); - modePairStatField.put("CURVATURERATIO", "curvature_ratio"); - } - - static { - mtd2dStatField.put("2D_AREA", "area"); - mtd2dStatField.put("2D_CENTROID_X", "centroid_x"); - mtd2dStatField.put("2D_CENTROID_Y", "centroid_y"); - mtd2dStatField.put("2D_CENTROID_LAT", "centroid_lat"); - mtd2dStatField.put("2D_CENTROID_LON", "centroid_lon"); - mtd2dStatField.put("2D_AXIS_ANG", "axis_ang"); - mtd2dStatField.put("2D_INTENSITY_10", "intensity_10"); - mtd2dStatField.put("2D_INTENSITY_25", "intensity_25"); - mtd2dStatField.put("2D_INTENSITY_50", "intensity_50"); - mtd2dStatField.put("2D_INTENSITY_75", "intensity_75"); - mtd2dStatField.put("2D_INTENSITY_90", "intensity_90"); - mtd2dStatField.put("2D_INTENSITY_N", "intensity_nn"); - } - - static { - mtd3dSingleStatField.put("3D_CENTROID_X", "centroid_x"); - mtd3dSingleStatField.put("3D_CENTROID_Y", "centroid_y"); - mtd3dSingleStatField.put("3D_CENTROID_T", "centroid_t"); - mtd3dSingleStatField.put("3D_CENTROID_LAT", "centroid_lat"); - mtd3dSingleStatField.put("3D_CENTROID_LON", "centroid_lon"); - mtd3dSingleStatField.put("3D_X_DOT", "x_dot"); - mtd3dSingleStatField.put("3D_Y_DOT", "y_dot"); - mtd3dSingleStatField.put("3D_AXIS_ANG", "axis_ang"); - mtd3dSingleStatField.put("3D_VOLUME", "volume"); - mtd3dSingleStatField.put("3D_START_TIME", "start_time"); - mtd3dSingleStatField.put("3D_END_TIME", "end_time"); - mtd3dSingleStatField.put("3D_DURATION", "end_time-start_time"); - mtd3dSingleStatField.put("3D_CDIST_TRAVELLED", "cdist_travelled"); - mtd3dSingleStatField.put("3D_INTENSITY_10", "intensity_10"); - mtd3dSingleStatField.put("3D_INTENSITY_25", "intensity_25"); - mtd3dSingleStatField.put("3D_INTENSITY_50", "intensity_50"); - mtd3dSingleStatField.put("3D_INTENSITY_75", "intensity_75"); - mtd3dSingleStatField.put("3D_INTENSITY_90", "intensity_90"); - mtd3dSingleStatField.put("3D_INTENSITY_N", "intensity_nn"); - } - - static { - mtd3dPairStatField.put("3D_SPACE_CENTROID_DIST", "space_centroid_dist"); - mtd3dPairStatField.put("3D_TIME_CENTROID_DELTA", "time_centroid_delta"); - mtd3dPairStatField.put("3D_AXIS_DIFF", "axis_diff"); - mtd3dPairStatField.put("3D_SPEED_DELTA", "speed_delta"); - mtd3dPairStatField.put("3D_DIRECTION_DIFF", "direction_diff"); - mtd3dPairStatField.put("3D_VOLUME_RATIO", "volume_ratio"); - mtd3dPairStatField.put("3D_START_TIME_DELTA", "start_time_delta"); - mtd3dPairStatField.put("3D_END_TIME_DELTA", "end_time_delta"); - mtd3dPairStatField.put("3D_INTERSECTION_VOLUME", "intersection_volume"); - mtd3dPairStatField.put("3D_DURATION_DIFF", "duration_diff"); - mtd3dPairStatField.put("3D_INTEREST", "interest"); - } - - static { - statsPerc.put("FCST_PERC", new String[]{}); - statsPerc.put("OBS_PERC", new String[]{}); - } - - - static { - calcStatCTC.add("BASER"); - calcStatCTC.add("BASER"); - calcStatCTC.add("ACC"); - calcStatCTC.add("FBIAS"); - calcStatCTC.add("PODY"); - calcStatCTC.add("POFD"); - calcStatCTC.add("PODN"); - calcStatCTC.add("FAR"); - calcStatCTC.add("CSI"); - calcStatCTC.add("GSS"); - } - - private MVUtil() { - throw new IllegalStateException("Utility class"); - } - - /** - * Build a list of strings representing consecutive dates between the input dates start and end, - * incrementing by incr number of seconds. It is assumed that the format of start and end is - * given by the java date format string format. The output dates will have the same format. - * - * @param start Beginning date, given in the format specified by the format input - * @param end End date, given in the format specified by the format input - * @param incr Number of seconds to increment between successive dates in the list - * @param format Java date format string, describing input and output dates - * @return List of date strings - */ - public static List buildDateList( - final String start, final String end, final int incr, - final String format) { - SimpleDateFormat formatDate = new SimpleDateFormat(format, Locale.US); - formatDate.setTimeZone(TimeZone.getTimeZone("UTC")); - List listDates = new ArrayList<>(); - - try { - Date dateStart = formatDate.parse(start); - Date dateEnd = formatDate.parse(end); - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - cal.setTime(dateStart); - - while ((incr > 0 && cal.getTime().getTime() <= dateEnd.getTime()) - || (incr < 0 && cal.getTime().getTime() >= dateEnd.getTime())) { - listDates.add(formatDate.format(cal.getTime())); - cal.add(Calendar.SECOND, incr); - } - - } catch (ParseException e) { - errorStream - .print(" ** ERROR: caught " + e.getClass() + " in buildDateList(): " + e.getMessage()); - - } - return listDates; - } - - /** - * Wrap the buildDateList implementation above, by parsing the specified node for start, end, incr - * and format and then returning the list of dates. - * - * @param node MVNode to parse for the date list parameters - * @return List of date strings - */ - public static List buildDateList(final MVNode node, final PrintStream printStream) - throws ValidationException { - String strStart = ""; - String strEnd = ""; - int intInc = 0; - String strFormat = ""; - - for (int j = 0; j < node.children.length; j++) { - MVNode nodeChild = node.children[j]; - if (nodeChild.tag.equals("inc")) { - intInc = Integer.parseInt(nodeChild.value); - } else if (nodeChild.tag.equals("format")) { - strFormat = nodeChild.value; - } - } - for (int j = 0; j < node.children.length; j++) { - MVNode nodeChild = node.children[j]; - if (nodeChild.tag.equals("start")) { - if (0 < nodeChild.children.length) { - strStart = parseDateOffset(nodeChild.children[0], strFormat); - } else { - strStart = nodeChild.value; - } - } else if (nodeChild.tag.equals("end")) { - strEnd = (0 < nodeChild.children.length - ? parseDateOffset(nodeChild.children[0], strFormat) : nodeChild.value); - } - } - - return buildDateList(strStart, strEnd, intInc, strFormat); - } - - /** - * Build a String representation of the date specified by the input {@link MVNode}. - * The offset is taken either from the current date (default) or from the date specified by the - * input date. - * - * @param node MVNode structure specifying the offset - * @param format (optional) String representation of the input/output date formats - * @param date (optional) String representation of the date from which to offset - * @return String representation of the offset date - */ - public static String parseDateOffset(final MVNode node, final String format, final String date) { - int intOffset = 0; - int intHour = 0; - - for (int i = 0; i < node.children.length; i++) { - MVNode nodeChild = node.children[i]; - if (nodeChild.tag.equals("day_offset")) { - intOffset = Integer.parseInt(nodeChild.value); - } else if (nodeChild.tag.equals("hour")) { - intHour = Integer.parseInt(nodeChild.value); - } - } - - SimpleDateFormat formatOffset = new SimpleDateFormat(format, Locale.US); - formatOffset.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - try { - cal.setTime(formatOffset.parse(date)); - } catch (Exception e) { - } - cal.set(Calendar.HOUR_OF_DAY, intHour); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.add(Calendar.DATE, intOffset); - - return formatOffset.format(cal.getTime()); - } - - public static String parseDateOffset(final MVNode node, final String format) throws ValidationException { - return parseDateOffset(node, format, null); - } - - - /** - * Concatenate the elements of the input list with surrounding ticks and separated by commas for - * use in the where clause of a SQL query. For example, the function call - * buildValueList(new String[]{"a", "bb", "c"}) will return the string "'a', 'bb', - * 'c'". - * - * @param values The list of values to be concatenated - * @return The string of concatenated values for use in a SQL where clause - */ - public static String buildValueList(final String[] values) { - String[] localValues; - if (values != null && values.length > 0) { - List newValues = new ArrayList<>(); - for (String value : values) { - if (value.contains(GROUP_SEPARATOR)) { - boolean matchFound = patDateTime.matcher(value).find(); - if(matchFound) { - //this is a date - need to use patterns - Matcher m = patDateTime.matcher(value); - while (m.find()) { - newValues.add(m.group()); + private static final org.apache.logging.log4j.Logger logger = LogManager.getLogger("MVUtil"); + + + public static final Pattern thresh = Pattern.compile("([<>=!]{1,2})(\\d*(?:\\.\\d+)?)"); + public static final Pattern lev = Pattern.compile("(\\w)(\\d+)(?:-(\\d+))?"); + public static final Pattern interpPnts = Pattern.compile("\\d+?"); + public static final Pattern modeSingle = Pattern.compile("\\s+h\\.([^,]+),"); + public static final Pattern prob = Pattern.compile("PROB\\(([\\w\\d]+)([<>=]+)([^\\)]+)\\)"); + private static final Pattern plotTmpl = Pattern.compile("\\{((\\w+)(?:\\?[^}]*)?)\\}"); + private static final Pattern tag = Pattern.compile("([\\w\\d]+)(?:\\s*\\?(.*))?"); + + + public static final Pattern patModeSingleObjectId = Pattern.compile("^(C?[FO]\\d{3})$"); + public static final Pattern patModePairObjectId = Pattern.compile("^(C?F\\d{3})_(C?O\\d{3})$"); + public static final Pattern patDateTime = Pattern.compile("(\\d{4})-(\\d{2})-(\\d{2}) (\\d{2}):(\\d{2}):(\\d{2})"); + + public static final DateTimeFormatter APP_DATE_FORMATTER + = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + public final static String LINE_SEPARATOR = System.getProperty("line.separator"); + /** + * Parse the input mode statistic, which is assumed to have the form SSSS_FFF, where SSSS is the + * name of a mode statistic with arbitrary length and FFF is a three character flag indicator + * string. + */ + public static final Pattern _patModeStat = Pattern.compile("([^_]+)(?:_\\w{3})?_(\\w{2,3})"); + public static final String CTC = "ctc"; //Contingency Table Statistics + public static final String MCTC = "mctc"; //Multi-category Contingency Table Count + public static final String NBRCTC = "nbrctc"; //Contingency Table Statistics + public static final String SL1L2 = "sl1l2"; //Scalar partial sums + public static final String GRAD = "grad"; //Gradient partial sums + public static final String SAL1L2 = "sal1l2"; // Scalar anomaly partial sums + public static final String SSVAR = "ssvar"; // Spread/Skill Variance + public static final String PCT = "pct"; //Contingency Table Counts for Probabilistic forecasts + public static final String NBRCNT = "nbrcnt"; // Neighborhood Continuous Statistics + public static final String VL1L2 = "vl1l2"; // Vector Partial Sum + public static final String VAL1L2 = "val1l2"; // Vector Anomaly Partial Sum + public static final String VCNT = "vcnt"; // Vector Continuous Statistics + public static final String ECNT = "ecnt"; // Ensemble Continuous Statistics + public static final String RPS = "rps"; // Ranked Probability Score Statistics + public static final Map statsEnscnt = new HashMap<>(); + public static final Map statsMpr = new HashMap<>(); + public static final Map statsDmap = new HashMap<>(); + public static final Map statsOrank = new HashMap<>(); + public static final Map statsCnt = new HashMap<>(); + public static final Map statsVcnt = new HashMap<>(); + public static final Map statsEcnt = new HashMap<>(); + public static final Map statsSsvar = new HashMap<>(); + public static final Map statsCts = new HashMap<>(); + public static final Map statsNbrcts = new HashMap<>(); + public static final Map statsNbrcnt = new HashMap<>(); + public static final Map statsPstd = new HashMap<>(); + public static final Map statsMcts = new HashMap<>(); + public static final Map statsPhist = new HashMap<>(); + public static final Map statsRhist = new HashMap<>(); + public static final Map statsVl1l2 = new HashMap<>(); + public static final Map statsVal1l2 = new HashMap<>(); + public static final Map statsPerc = new HashMap<>(); + public static final Map modeSingleStatField = new HashMap<>(); + public static final Map mtd3dSingleStatField = new HashMap<>(); + public static final Map mtd2dStatField = new HashMap<>(); + public static final Map statsRps = new HashMap<>(); + public static final Map statsCtc = new HashMap<>(); + public static final Map statsNbrctc = new HashMap<>(); + public static final Map statsPct = new HashMap<>(); + public static final Map statsSl1l2 = new HashMap<>(); + public static final Map statsSal1l2 = new HashMap<>(); + public static final Map statsGrad = new HashMap<>(); + public static final Map statsMctc = new HashMap<>(); + public static final Map statsSsidx = new HashMap<>(); + public static final Map statsSeeps = new HashMap<>(); + + + public static final int MAX_STR_LEN = 500; + + + public static final List modeRatioField = new ArrayList<>(); + public static final List mtdRatioField = new ArrayList<>(); + + + public static final Map modePairStatField = new HashMap<>(); + public static final Map mtd3dPairStatField = new HashMap<>(); + public static final List calcStatCTC = new ArrayList<>(); + public static final String DB_DATE_MS = "yyyy-MM-dd HH:mm:ss.S"; + public static final SimpleDateFormat PLOT_FORMAT = new SimpleDateFormat("yyyyMMddHH"); + + + public static final Pattern patRTmpl = Pattern.compile("#<(\\w+)>#"); + + public static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("0.000"); + public static final String PYTHON = "Python"; + public static final String RSCRIPT = "Rscript"; + + /* + * variable length group data indices for lines with an arbitrary number of fields + * - index of field containing number of sets + * - index of first repeating field(s) + * - number of fields in each repeating set + */ + public static final Map lengthGroupIndices = new HashMap<>(); + public static final Map alphaLineTypes = new HashMap<>(); + + public static final Map covThreshLineTypes = new HashMap<>(); + + public static final PrintStream errorStream = IoBuilder.forLogger(MVUtil.class) + .setLevel(org.apache.logging.log4j.Level.INFO) + .buildPrintStream(); + public static final String DEFAULT_DATABASE_GROUP = "NO GROUP"; + + public static final String SEPARATOR = ","; + public static final String GROUP_SEPARATOR = ":"; + + public static final String[] lineTypes = new String[]{ + "fho", + "ctc", + "cts", + "mctc", + "mcts", + "cnt", + "sl1l2", + "sal1l2", + "vl1l2", + "val1l2", + "pct", + "pstd", + "pjc", + "prc", + "eclv", + "mpr", + "nbrctc", + "nbrcts", + "nbrcnt", + "isc", + "rhist", + "phist", + "orank", + "ssvar", + "grad", + "vcnt", + "relp", + "ecnt", + "dmap", + "rps", + "seeps" + }; + + static { + + modeRatioField.add("RATIO_FSA_ASA"); + modeRatioField.add("RATIO_OSA_ASA"); + modeRatioField.add("RATIO_ASM_ASA"); + modeRatioField.add("RATIO_ASU_ASA"); + modeRatioField.add("RATIO_FSM_FSA"); + modeRatioField.add("RATIO_FSU_FSA"); + modeRatioField.add("RATIO_OSM_OSA"); + modeRatioField.add("RATIO_OSU_OSA"); + modeRatioField.add("RATIO_FSM_ASM"); + modeRatioField.add("RATIO_OSM_ASM"); + modeRatioField.add("RATIO_FSU_ASU"); + modeRatioField.add("RATIO_OSU_ASU"); + modeRatioField.add("RATIO_FSA_AAA"); + modeRatioField.add("RATIO_OSA_AAA"); + modeRatioField.add("RATIO_FSA_FAA"); + modeRatioField.add("RATIO_FCA_FAA"); + modeRatioField.add("RATIO_OSA_OAA"); + modeRatioField.add("RATIO_OCA_OAA"); + modeRatioField.add("RATIO_FCA_ACA"); + modeRatioField.add("RATIO_OCA_ACA"); + modeRatioField.add("RATIO_FSA_OSA"); + modeRatioField.add("RATIO_OSA_FSA"); + modeRatioField.add("RATIO_ACA_ASA"); + modeRatioField.add("RATIO_ASA_ACA"); + modeRatioField.add("RATIO_FCA_FSA"); + modeRatioField.add("RATIO_FSA_FCA"); + modeRatioField.add("RATIO_OCA_OSA"); + modeRatioField.add("RATIO_OSA_OCA"); + + modeRatioField.add("OBJHITS"); + modeRatioField.add("OBJMISSES"); + modeRatioField.add("OBJFAS"); + modeRatioField.add("OBJCSI"); + modeRatioField.add("OBJPODY"); + modeRatioField.add("OBJFAR"); + + modeRatioField.add("AREARAT_FSA_ASA"); + modeRatioField.add("AREARAT_OSA_ASA"); + modeRatioField.add("AREARAT_ASM_ASA"); + modeRatioField.add("AREARAT_ASU_ASA"); + modeRatioField.add("AREARAT_FSM_FSA"); + modeRatioField.add("AREARAT_FSU_FSA"); + modeRatioField.add("AREARAT_OSM_OSA"); + modeRatioField.add("AREARAT_OSU_OSA"); + modeRatioField.add("AREARAT_FSM_ASM"); + modeRatioField.add("AREARAT_OSM_ASM"); + modeRatioField.add("AREARAT_FSU_ASU"); + modeRatioField.add("AREARAT_OSU_ASU"); + modeRatioField.add("AREARAT_FSA_AAA"); + modeRatioField.add("AREARAT_OSA_AAA"); + modeRatioField.add("AREARAT_FSA_FAA"); + modeRatioField.add("AREARAT_FCA_FAA"); + modeRatioField.add("AREARAT_OSA_OAA"); + modeRatioField.add("AREARAT_OCA_OAA"); + modeRatioField.add("AREARAT_FCA_ACA"); + modeRatioField.add("AREARAT_OCA_ACA"); + modeRatioField.add("AREARAT_FSA_OSA"); + modeRatioField.add("AREARAT_OSA_FSA"); + modeRatioField.add("AREARAT_ACA_ASA"); + modeRatioField.add("AREARAT_ASA_ACA"); + modeRatioField.add("AREARAT_FCA_FSA"); + modeRatioField.add("AREARAT_FSA_FCA"); + modeRatioField.add("AREARAT_OCA_OSA"); + modeRatioField.add("AREARAT_OSA_OCA"); + + modeRatioField.add("OBJAHITS"); + modeRatioField.add("OBJAMISSES"); + modeRatioField.add("OBJAFAS"); + modeRatioField.add("OBJACSI"); + modeRatioField.add("OBJAPODY"); + modeRatioField.add("OBJAFAR"); + } + + + static { + + mtdRatioField.add("2d_RATIO_FSA_ASA"); + mtdRatioField.add("2d_RATIO_OSA_ASA"); + mtdRatioField.add("2d_RATIO_ASM_ASA"); + mtdRatioField.add("2d_RATIO_ASU_ASA"); + mtdRatioField.add("2d_RATIO_FSM_FSA"); + mtdRatioField.add("2d_RATIO_FSU_FSA"); + mtdRatioField.add("2d_RATIO_OSM_OSA"); + mtdRatioField.add("2d_RATIO_OSU_OSA"); + mtdRatioField.add("2d_RATIO_FSM_ASM"); + mtdRatioField.add("2d_RATIO_OSM_ASM"); + mtdRatioField.add("2d_RATIO_FSU_ASU"); + mtdRatioField.add("2d_RATIO_OSU_ASU"); + mtdRatioField.add("2d_RATIO_FSA_AAA"); + mtdRatioField.add("2d_RATIO_OSA_AAA"); + mtdRatioField.add("2d_RATIO_FSA_FAA"); + mtdRatioField.add("2d_RATIO_FCA_FAA"); + mtdRatioField.add("2d_RATIO_OSA_OAA"); + mtdRatioField.add("2d_RATIO_OCA_OAA"); + mtdRatioField.add("2d_RATIO_FCA_ACA"); + mtdRatioField.add("2d_RATIO_OCA_ACA"); + mtdRatioField.add("2d_RATIO_FSA_OSA"); + mtdRatioField.add("2d_RATIO_OSA_FSA"); + mtdRatioField.add("2d_RATIO_ACA_ASA"); + mtdRatioField.add("2d_RATIO_ASA_ACA"); + mtdRatioField.add("2d_RATIO_FCA_FSA"); + mtdRatioField.add("2d_RATIO_FSA_FCA"); + mtdRatioField.add("2d_RATIO_OCA_OSA"); + mtdRatioField.add("2d_RATIO_OSA_OCA"); + + + mtdRatioField.add("3d_RATIO_FSA_ASA"); + mtdRatioField.add("3d_RATIO_OSA_ASA"); + mtdRatioField.add("3d_RATIO_ASM_ASA"); + mtdRatioField.add("3d_RATIO_ASU_ASA"); + mtdRatioField.add("3d_RATIO_FSM_FSA"); + mtdRatioField.add("3d_RATIO_FSU_FSA"); + mtdRatioField.add("3d_RATIO_OSM_OSA"); + mtdRatioField.add("3d_RATIO_OSU_OSA"); + mtdRatioField.add("3d_RATIO_FSM_ASM"); + mtdRatioField.add("3d_RATIO_OSM_ASM"); + mtdRatioField.add("3d_RATIO_FSU_ASU"); + mtdRatioField.add("3d_RATIO_OSU_ASU"); + mtdRatioField.add("3d_RATIO_FSA_AAA"); + mtdRatioField.add("3d_RATIO_OSA_AAA"); + mtdRatioField.add("3d_RATIO_FSA_FAA"); + mtdRatioField.add("3d_RATIO_FCA_FAA"); + mtdRatioField.add("3d_RATIO_OSA_OAA"); + mtdRatioField.add("3d_RATIO_OCA_OAA"); + mtdRatioField.add("3d_RATIO_FCA_ACA"); + mtdRatioField.add("3d_RATIO_OCA_ACA"); + mtdRatioField.add("3d_RATIO_FSA_OSA"); + mtdRatioField.add("3d_RATIO_OSA_FSA"); + mtdRatioField.add("3d_RATIO_ACA_ASA"); + mtdRatioField.add("3d_RATIO_ASA_ACA"); + mtdRatioField.add("3d_RATIO_FCA_FSA"); + mtdRatioField.add("3d_RATIO_FSA_FCA"); + mtdRatioField.add("3d_RATIO_OCA_OSA"); + mtdRatioField.add("3d_RATIO_OSA_OCA"); + + mtdRatioField.add("2d_OBJHITS"); + mtdRatioField.add("2d_OBJMISSES"); + mtdRatioField.add("2d_OBJFAS"); + mtdRatioField.add("2d_OBJCSI"); + mtdRatioField.add("2d_OBJPODY"); + mtdRatioField.add("2d_OBJFAR"); + mtdRatioField.add("3d_OBJHITS"); + mtdRatioField.add("3d_OBJMISSES"); + mtdRatioField.add("3d_OBJFAS"); + mtdRatioField.add("3d_OBJCSI"); + mtdRatioField.add("3d_OBJPODY"); + mtdRatioField.add("3d_OBJFAR"); + + mtdRatioField.add("3d_VOLRAT_FSA_ASA"); + mtdRatioField.add("3d_VOLRAT_OSA_ASA"); + mtdRatioField.add("3d_VOLRAT_ASM_ASA"); + mtdRatioField.add("3d_VOLRAT_ASU_ASA"); + mtdRatioField.add("3d_VOLRAT_FSM_FSA"); + mtdRatioField.add("3d_VOLRAT_FSU_FSA"); + mtdRatioField.add("3d_VOLRAT_OSM_OSA"); + mtdRatioField.add("3d_VOLRAT_OSU_OSA"); + mtdRatioField.add("3d_VOLRAT_FSM_ASM"); + mtdRatioField.add("3d_VOLRAT_OSM_ASM"); + mtdRatioField.add("3d_VOLRAT_FSU_ASU"); + mtdRatioField.add("3d_VOLRAT_OSU_ASU"); + mtdRatioField.add("3d_VOLRAT_FSA_AAA"); + mtdRatioField.add("3d_VOLRAT_OSA_AAA"); + mtdRatioField.add("3d_VOLRAT_FSA_FAA"); + mtdRatioField.add("3d_VOLRAT_FCA_FAA"); + mtdRatioField.add("3d_VOLRAT_OSA_OAA"); + mtdRatioField.add("3d_VOLRAT_OCA_OAA"); + mtdRatioField.add("3d_VOLRAT_FCA_ACA"); + mtdRatioField.add("3d_VOLRAT_OCA_ACA"); + mtdRatioField.add("3d_VOLRAT_FSA_OSA"); + mtdRatioField.add("3d_VOLRAT_OSA_FSA"); + mtdRatioField.add("3d_VOLRAT_ACA_ASA"); + mtdRatioField.add("3d_VOLRAT_ASA_ACA"); + mtdRatioField.add("3d_VOLRAT_FCA_FSA"); + mtdRatioField.add("3d_VOLRAT_FSA_FCA"); + mtdRatioField.add("3d_VOLRAT_OCA_OSA"); + mtdRatioField.add("3d_VOLRAT_OSA_OCA"); + + mtdRatioField.add("2d_AREARAT_FSA_ASA"); + mtdRatioField.add("2d_AREARAT_OSA_ASA"); + mtdRatioField.add("2d_AREARAT_ASM_ASA"); + mtdRatioField.add("2d_AREARAT_ASU_ASA"); + mtdRatioField.add("2d_AREARAT_FSM_FSA"); + mtdRatioField.add("2d_AREARAT_FSU_FSA"); + mtdRatioField.add("2d_AREARAT_OSM_OSA"); + mtdRatioField.add("2d_AREARAT_OSU_OSA"); + mtdRatioField.add("2d_AREARAT_FSM_ASM"); + mtdRatioField.add("2d_AREARAT_OSM_ASM"); + mtdRatioField.add("2d_AREARAT_FSU_ASU"); + mtdRatioField.add("2d_AREARAT_OSU_ASU"); + mtdRatioField.add("2d_AREARAT_FSA_AAA"); + mtdRatioField.add("2d_AREARAT_OSA_AAA"); + mtdRatioField.add("2d_AREARAT_FSA_FAA"); + mtdRatioField.add("2d_AREARAT_FCA_FAA"); + mtdRatioField.add("2d_AREARAT_OSA_OAA"); + mtdRatioField.add("2d_AREARAT_OCA_OAA"); + mtdRatioField.add("2d_AREARAT_FCA_ACA"); + mtdRatioField.add("2d_AREARAT_OCA_ACA"); + mtdRatioField.add("2d_AREARAT_FSA_OSA"); + mtdRatioField.add("2d_AREARAT_OSA_FSA"); + mtdRatioField.add("2d_AREARAT_ACA_ASA"); + mtdRatioField.add("2d_AREARAT_ASA_ACA"); + mtdRatioField.add("2d_AREARAT_FCA_FSA"); + mtdRatioField.add("2d_AREARAT_FSA_FCA"); + mtdRatioField.add("2d_AREARAT_OCA_OSA"); + mtdRatioField.add("2d_AREARAT_OSA_OCA"); + + mtdRatioField.add("3d_OBJVHITS"); + mtdRatioField.add("3d_OBJVMISSES"); + mtdRatioField.add("3d_OBJVFAS"); + mtdRatioField.add("3d_OBJVCSI"); + mtdRatioField.add("3d_OBJVPODY"); + mtdRatioField.add("3d_OBJVFAR"); + mtdRatioField.add("3d_OBJVHITS"); + mtdRatioField.add("2d_OBJAMISSES"); + mtdRatioField.add("2d_OBJAFAS"); + mtdRatioField.add("2d_OBJACSI"); + mtdRatioField.add("2d_OBJAPODY"); + mtdRatioField.add("2d_OBJAFAR"); + } + + + static { + covThreshLineTypes.put("NBRCTC", Boolean.TRUE); + covThreshLineTypes.put("NBRCTS", Boolean.TRUE); + covThreshLineTypes.put("PCT", Boolean.TRUE); + covThreshLineTypes.put("PSTD", Boolean.TRUE); + covThreshLineTypes.put("PJC", Boolean.TRUE); + covThreshLineTypes.put("PRC", Boolean.TRUE); + } + + static { + alphaLineTypes.put("CTS", Boolean.TRUE); + alphaLineTypes.put("NBRCTS", Boolean.TRUE); + alphaLineTypes.put("NBRCNT", Boolean.TRUE); + alphaLineTypes.put("MCTS", Boolean.TRUE); + alphaLineTypes.put("SSVAR", Boolean.TRUE); + alphaLineTypes.put("VCNT", Boolean.TRUE); + alphaLineTypes.put("DMAP", Boolean.TRUE); + alphaLineTypes.put("RPS", Boolean.TRUE); + + alphaLineTypes.put("CNT", Boolean.FALSE); + alphaLineTypes.put("PSTD", Boolean.FALSE); + } + + + static { + lengthGroupIndices.put("PCT", new int[]{23, 24, 3}); + lengthGroupIndices.put("PSTD", new int[]{23, 33, 1}); + lengthGroupIndices.put("PJC", new int[]{23, 24, 7}); + lengthGroupIndices.put("PRC", new int[]{23, 24, 3}); + lengthGroupIndices.put("MCTC", new int[]{23, 24, 1}); + lengthGroupIndices.put("RHIST", new int[]{23, 23, 1}); + lengthGroupIndices.put("PHIST", new int[]{24, 25, 1}); + lengthGroupIndices.put("RELP", new int[]{23, 24, 1}); + lengthGroupIndices.put("ORANK", new int[]{33, 34, 1}); + lengthGroupIndices.put("ECLV", new int[]{25, 26, 2}); + } + + static { + statsEnscnt.put("ENS_RPSF", new String[]{}); + statsEnscnt.put("ENS_RPSCL", new String[]{}); + statsEnscnt.put("ENS_RPSS", new String[]{}); + statsEnscnt.put("ENS_CRPSF", new String[]{}); + statsEnscnt.put("ENS_CRPSCL", new String[]{}); + statsEnscnt.put("ENS_CRPSS", new String[]{}); + } + + static { + statsMpr.put("MPR_FCST", new String[]{""}); + statsMpr.put("MPR_OBS", new String[]{""}); + statsMpr.put("MPR_CLIMO", new String[]{""}); + } + + static { + statsDmap.put("DMAP_FBIAS", new String[]{""}); + statsDmap.put("DMAP_BADDELEY", new String[]{""}); + statsDmap.put("DMAP_HAUSDORFF", new String[]{""}); + statsDmap.put("DMAP_MED_FO", new String[]{""}); + statsDmap.put("DMAP_MED_OF", new String[]{""}); + statsDmap.put("DMAP_MED_MIN", new String[]{""}); + statsDmap.put("DMAP_MED_MAX", new String[]{""}); + statsDmap.put("DMAP_MED_MEAN", new String[]{""}); + statsDmap.put("DMAP_FOM_FO", new String[]{""}); + statsDmap.put("DMAP_FOM_OF", new String[]{""}); + statsDmap.put("DMAP_FOM_MIN", new String[]{""}); + statsDmap.put("DMAP_FOM_MAX", new String[]{""}); + statsDmap.put("DMAP_FOM_MEAN", new String[]{""}); + statsDmap.put("DMAP_ZHU_FO", new String[]{""}); + statsDmap.put("DMAP_ZHU_OF", new String[]{""}); + statsDmap.put("DMAP_ZHU_MIN", new String[]{""}); + statsDmap.put("DMAP_ZHU_MAX", new String[]{""}); + statsDmap.put("DMAP_ZHU_MEAN", new String[]{""}); + statsDmap.put("DMAP_G", new String[]{""}); + statsDmap.put("DMAP_GBETA", new String[]{""}); + } + + static { + statsRps.put("RPS_REL", new String[]{""}); + statsRps.put("RPS_RES", new String[]{""}); + statsRps.put("RPS_UNC", new String[]{""}); + statsRps.put("RPS", new String[]{RPS}); + statsRps.put("RPSS", new String[]{RPS}); + statsRps.put("RPSS_SMPL", new String[]{""}); + statsRps.put("RPS_COMP", new String[]{RPS}); + statsRps.put("RPS_TOTAL", new String[]{RPS}); + } + + static { + statsCtc.put("CTC_FY_OY", new String[]{CTC}); + statsCtc.put("CTC_FY_ON", new String[]{CTC}); + statsCtc.put("CTC_FN_OY", new String[]{CTC}); + statsCtc.put("CTC_FN_ON", new String[]{CTC}); + statsCtc.put("CTC_OY", new String[]{CTC}); + statsCtc.put("CTC_ON", new String[]{CTC}); + statsCtc.put("CTC_FY", new String[]{CTC}); + statsCtc.put("CTC_FN", new String[]{CTC}); + statsCtc.put("CTC_TOTAL", new String[]{CTC}); + } + + static { + statsOrank.put("ORANK_PIT", new String[]{""}); + statsOrank.put("ORANK_OBS_RANK", new String[]{""}); + statsOrank.put("ORANK_ENS_MEAN", new String[]{""}); + statsOrank.put("ORANK_SPREAD", new String[]{""}); + statsOrank.put("ORANK_CLIMO_MEAN", new String[]{""}); + statsOrank.put("ORANK_ENS_MEAN_OERR", new String[]{""}); + statsOrank.put("ORANK_SPREAD_OERR", new String[]{""}); + statsOrank.put("ORANK_SPREAD_PLUS_OERR", new String[]{""}); + statsOrank.put("ORANK_CLIMO_STDEV", new String[]{""}); + } + + static { + statsCnt.put("FBAR", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("FSTDEV", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("OBAR", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("OSTDEV", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("PR_CORR", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("SP_CORR", new String[]{}); + statsCnt.put("KT_CORR", new String[]{}); + statsCnt.put("ME", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("ESTDEV", new String[]{"nc", "bc", SL1L2}); + statsCnt.put("MBIAS", new String[]{"bc", SL1L2}); + statsCnt.put("MAE", new String[]{"bc", SL1L2}); + statsCnt.put("MSE", new String[]{"bc", SL1L2}); + statsCnt.put("BCMSE", new String[]{"bc", SL1L2}); + statsCnt.put("BCRMSE", new String[]{"bc", SL1L2}); + statsCnt.put("RMSE", new String[]{"bc", SL1L2}); + statsCnt.put("FGBAR", new String[]{"bc", GRAD}); + statsCnt.put("OGBAR", new String[]{"bc", GRAD}); + statsCnt.put("MGBAR", new String[]{"bc", GRAD}); + statsCnt.put("EGBAR", new String[]{"bc", GRAD}); + statsCnt.put("S1", new String[]{"bc", GRAD}); + statsCnt.put("S1_OG", new String[]{"bc", GRAD}); + statsCnt.put("FGOG_RATIO", new String[]{"bc", GRAD}); + statsCnt.put("E10", new String[]{"bc"}); + statsCnt.put("E25", new String[]{"bc"}); + statsCnt.put("E50", new String[]{"bc"}); + statsCnt.put("E75", new String[]{"bc"}); + statsCnt.put("E90", new String[]{"bc"}); + statsCnt.put("IQR", new String[]{"bc"}); + statsCnt.put("MAD", new String[]{"bc"}); + //statsCnt.put("PAC", new String[]{"bc"}); + statsCnt.put("ANOM_CORR", new String[]{"bc", SAL1L2}); + statsCnt.put("ANOM_CORR_UNCNTR", new String[]{"bc", SAL1L2}); + statsCnt.put("ME2", new String[]{"bc", SL1L2}); + statsCnt.put("MSESS", new String[]{"bc", SL1L2}); + statsCnt.put("RMSFA", new String[]{"bc", SAL1L2}); + statsCnt.put("RMSOA", new String[]{"bc", SAL1L2}); + statsCnt.put("SI", new String[]{"bc", SL1L2}); + } + + static { + statsSl1l2.put("SL1L2_TOTAL", new String[]{SL1L2}); + } + + static { + statsSal1l2.put("SAL1L2_TOTAL", new String[]{SAL1L2}); + } + + static { + statsGrad.put("GRAD_TOTAL", new String[]{GRAD}); + } + + static { + statsVcnt.put("VCNT_FBAR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_OBAR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_FS_RMS", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_OS_RMS", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_MSVE", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_RMSVE", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_FSTDEV", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_OSTDEV", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_FDIR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_ODIR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_FBAR_SPEED", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_OBAR_SPEED", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_VDIFF_SPEED", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_VDIFF_DIR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_SPEED_ERR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_SPEED_ABSERR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_ERR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_ABSERR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_ANOM_CORR", new String[]{"nc", "bc", VCNT}); + statsVcnt.put("VCNT_ANOM_CORR_UNCNTR", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_ME", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_MAE", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_MSE", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_DIR_RMSE", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_TOTAL", new String[]{"bc", VCNT}); + statsVcnt.put("VCNT_TOTAL_DIR", new String[]{"bc", VCNT}); + + + } + + static { + statsEcnt.put("ECNT_CRPS", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPSS", new String[]{ECNT}); + statsEcnt.put("ECNT_IGN", new String[]{ECNT}); + statsEcnt.put("ECNT_ME", new String[]{ECNT}); + statsEcnt.put("ECNT_RMSE", new String[]{ECNT}); + statsEcnt.put("ECNT_SPREAD", new String[]{ECNT}); + statsEcnt.put("ECNT_ME_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_RMSE_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_SPREAD_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_SPREAD_PLUS_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPSCL", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPS_EMP", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPSCL_EMP", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPSS_EMP", new String[]{ECNT}); + statsEcnt.put("ECNT_CRPS_EMP_FAIR", new String[]{ECNT}); + statsEcnt.put("ECNT_SPREAD_MD", new String[]{ECNT}); + statsEcnt.put("ECNT_MAE", new String[]{ECNT}); + statsEcnt.put("ECNT_MAE_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_BIAS_RATIO", new String[]{ECNT}); + statsEcnt.put("ECNT_N_GE_OBS", new String[]{ECNT}); + statsEcnt.put("ECNT_ME_GE_OBS", new String[]{ECNT}); + statsEcnt.put("ECNT_N_LT_OBS", new String[]{ECNT}); + statsEcnt.put("ECNT_ME_LT_OBS", new String[]{ECNT}); + statsEcnt.put("ECNT_IGN_CONV_OERR", new String[]{ECNT}); + statsEcnt.put("ECNT_IGN_CORR_OERR", new String[]{ECNT}); + + statsEcnt.put("ECNT_TOTAL", new String[]{ECNT}); + } + + static { + statsSsvar.put("SSVAR_FBAR", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_FSTDEV", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_OBAR", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_OSTDEV", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_PR_CORR", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_ME", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_ESTDEV", new String[]{"nc", "bc", SSVAR}); + statsSsvar.put("SSVAR_MBIAS", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_MSE", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_BCMSE", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_BCRMSE", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_RMSE", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_ANOM_CORR", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_ME2", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_MSESS", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_Spread", new String[]{"bc", SSVAR}); + statsSsvar.put("SSVAR_TOTAL", new String[]{SSVAR}); + } + + + static { + statsCts.put("BASER", new String[]{"nc", "bc", CTC}); + statsCts.put("FMEAN", new String[]{"nc", "bc", CTC}); + statsCts.put("ACC", new String[]{"nc", "bc", CTC}); + statsCts.put("FBIAS", new String[]{"bc", CTC}); + statsCts.put("PODY", new String[]{"nc", "bc", CTC}); + statsCts.put("PODN", new String[]{"nc", "bc", CTC}); + statsCts.put("POFD", new String[]{"nc", "bc", CTC}); + statsCts.put("FAR", new String[]{"nc", "bc", CTC}); + statsCts.put("CSI", new String[]{"nc", "bc", CTC}); + statsCts.put("GSS", new String[]{"bc", CTC}); + statsCts.put("HK", new String[]{"nc", "bc", CTC}); + statsCts.put("HSS", new String[]{"bc", CTC}); + statsCts.put("ODDS", new String[]{"nc", "bc", CTC}); + statsCts.put("LODDS", new String[]{"nc", "bc", CTC}); + statsCts.put("ORSS", new String[]{"nc", "bc", CTC}); + statsCts.put("EDS", new String[]{"nc", "bc", CTC}); + statsCts.put("SEDS", new String[]{"nc", "bc", CTC}); + statsCts.put("EDI", new String[]{"nc", "bc", CTC}); + statsCts.put("SEDI", new String[]{"nc", "bc", CTC}); + statsCts.put("BAGSS", new String[]{"bc", CTC}); + statsCts.put("HSS_EC", new String[]{"bc", CTC}); + } + + static { + statsMctc.put("MCTS_HSS_EC", new String[]{"bc", MCTC}); + } + + static { + statsSsidx.put("SS_INDEX", new String[]{}); + } + + static { + statsSeeps.put("SEEPS_MEAN_FCST", new String[]{}); + statsSeeps.put("SEEPS_MEAN_OBS", new String[]{}); + statsSeeps.put("SEEPS", new String[]{}); + statsSeeps.put("SEEPS_S12", new String[]{}); + statsSeeps.put("SEEPS_S13", new String[]{}); + statsSeeps.put("SEEPS_S21", new String[]{}); + statsSeeps.put("SEEPS_S23", new String[]{}); + statsSeeps.put("SEEPS_S31", new String[]{}); + statsSeeps.put("SEEPS_S32", new String[]{}); + statsSeeps.put("SEEPS_PF1", new String[]{}); + statsSeeps.put("SEEPS_PF2", new String[]{}); + statsSeeps.put("SEEPS_PF3", new String[]{}); + statsSeeps.put("SEEPS_PV1", new String[]{}); + statsSeeps.put("SEEPS_PV2", new String[]{}); + statsSeeps.put("SEEPS_PV3", new String[]{}); + } + + + static { + statsNbrcts.put("NBR_BASER", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_FMEAN", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_ACC", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_FBIAS", new String[]{"bc", NBRCTC}); + statsNbrcts.put("NBR_PODY", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_PODN", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_POFD", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_FAR", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_CSI", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_GSS", new String[]{"bc", NBRCTC}); + statsNbrcts.put("NBR_HK", new String[]{"nc", "bc", NBRCTC}); + statsNbrcts.put("NBR_HSS", new String[]{"bc", NBRCTC}); + statsNbrcts.put("NBR_ODDS", new String[]{"nc", "bc", NBRCTC}); + } + + static { + statsNbrctc.put("NBR_CTC_TOTAL", new String[]{NBRCTC}); + } + + static { + statsNbrcnt.put("NBR_FBS", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_FSS", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_AFSS", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_UFSS", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_F_RATE", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_O_RATE", new String[]{"bc", NBRCNT}); + statsNbrcnt.put("NBR_CNT_TOTAL", new String[]{NBRCNT}); + } + + static { + statsPstd.put("PSTD_BASER", new String[]{"nc", PCT}); + statsPstd.put("PSTD_RELIABILITY", new String[]{PCT}); + statsPstd.put("PSTD_RESOLUTION", new String[]{PCT}); + statsPstd.put("PSTD_UNCERTAINTY", new String[]{PCT}); + statsPstd.put("PSTD_ROC_AUC", new String[]{PCT}); + statsPstd.put("PSTD_BRIER", new String[]{"nc", PCT}); + statsPstd.put("PSTD_BRIERCL", new String[]{}); + statsPstd.put("PSTD_BSS", new String[]{}); + statsPstd.put("PSTD_BSS_SMPL", new String[]{PCT}); + } + + static { + statsPct.put("PCT_TOTAL", new String[]{PCT}); + } + + static { + statsMcts.put("MCTS_ACC", new String[]{"nc", "bc"}); + statsMcts.put("MCTS_HK", new String[]{"bc"}); + statsMcts.put("MCTS_HSS", new String[]{"bc"}); + statsMcts.put("MCTS_GER", new String[]{"bc"}); + statsMcts.put("MCTS_HSS_EC", new String[]{"bc"}); + //statsMcts.put("MCTS_TOTAL", new String[]{MCTC}); + } + + static { + statsRhist.put("RHIST_CRPS", new String[]{}); + statsRhist.put("RHIST_CRPSS", new String[]{}); + statsRhist.put("RHIST_IGN", new String[]{}); + statsRhist.put("RHIST_SPREAD", new String[]{}); + } + + static { + statsVal1l2.put("VAL1L2_ANOM_CORR", new String[]{VAL1L2}); + statsVal1l2.put("VAL1L2_DIRA_ME", new String[]{VAL1L2}); + statsVal1l2.put("VAL1L2_DIRA_MAE", new String[]{VAL1L2}); + statsVal1l2.put("VAL1L2_DIRA_MSE", new String[]{VAL1L2}); + statsVal1l2.put("VAL1L2_TOTAL", new String[]{VAL1L2}); + } + + static { + statsVl1l2.put("VL1L2_UFBAR", new String[]{}); + statsVl1l2.put("VL1L2_VFBAR", new String[]{}); + statsVl1l2.put("VL1L2_UOBAR", new String[]{}); + statsVl1l2.put("VL1L2_VOBAR", new String[]{}); + statsVl1l2.put("VL1L2_BIAS", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_FVAR", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_OVAR", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_SPEED_ERR", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_RMSVE", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_SPEED_DIFF", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_DIR_ME", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_DIR_MAE", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_DIR_MSE", new String[]{VL1L2}); + statsVl1l2.put("VL1L2_TOTAL", new String[]{VL1L2}); + } + + public static final String COUNT = "COUNT(*)"; + + static { + modeSingleStatField.put("ACOV", "SUM(area)"); + modeSingleStatField.put("CNT", COUNT); + modeSingleStatField.put("CNTSUM", COUNT); + modeSingleStatField.put("CENTX", "centroid_x"); + modeSingleStatField.put("CENTY", "centroid_y"); + modeSingleStatField.put("CENTLAT", "centroid_lat"); + modeSingleStatField.put("CENTLON", "centroid_lon"); + modeSingleStatField.put("AXAVG", "axis_avg"); + modeSingleStatField.put("LEN", "length"); + modeSingleStatField.put("WID", "width"); + modeSingleStatField + .put("ASPECT", "IF((length/width) < (width/length), length/width, width/length)"); + modeSingleStatField.put("AREA", "area"); + modeSingleStatField.put("AREATHR", "area_thresh"); + modeSingleStatField.put("CURV", "curvature"); + modeSingleStatField.put("CURVX", "curvature_x"); + modeSingleStatField.put("CURVY", "curvature_y"); + modeSingleStatField.put("CPLX", "complexity"); + modeSingleStatField.put("INT10", "intensity_10"); + modeSingleStatField.put("INT25", "intensity_25"); + modeSingleStatField.put("INT50", "intensity_50"); + modeSingleStatField.put("INT75", "intensity_75"); + modeSingleStatField.put("INT90", "intensity_90"); + modeSingleStatField.put("INTN", "intensity_nn"); + modeSingleStatField.put("INTSUM", "intensity_sum"); + + } + + + static { + modePairStatField.put("CENTDIST", "centroid_dist"); + modePairStatField.put("BOUNDDIST", "boundary_dist"); + modePairStatField.put("HULLDIST", "convex_hull_dist"); + modePairStatField.put("ANGLEDIFF", "angle_diff"); + modePairStatField.put("AREARATIO", "area_ratio"); + modePairStatField.put("INTAREA", "intersection_area"); + modePairStatField.put("UNIONAREA", "union_area"); + modePairStatField.put("SYMDIFF", "symmetric_diff"); + modePairStatField.put("INTOVERAREA", "intersection_over_area"); + modePairStatField.put("CMPLXRATIO", "complexity_ratio"); + modePairStatField.put("PERCINTRATIO", "percentile_intensity_ratio"); + modePairStatField.put("INT", "interest"); + modePairStatField.put("MAXINT", "MAX(interest)"); + modePairStatField.put("MAXINTF", "MAX(interest)"); + modePairStatField.put("MAXINTO", "MAX(interest)"); + modePairStatField.put("ASPECTDIFF", "aspect_diff"); + modePairStatField.put("CURVATURERATIO", "curvature_ratio"); + } + + static { + mtd2dStatField.put("2D_AREA", "area"); + mtd2dStatField.put("2D_CENTROID_X", "centroid_x"); + mtd2dStatField.put("2D_CENTROID_Y", "centroid_y"); + mtd2dStatField.put("2D_CENTROID_LAT", "centroid_lat"); + mtd2dStatField.put("2D_CENTROID_LON", "centroid_lon"); + mtd2dStatField.put("2D_AXIS_ANG", "axis_ang"); + mtd2dStatField.put("2D_INTENSITY_10", "intensity_10"); + mtd2dStatField.put("2D_INTENSITY_25", "intensity_25"); + mtd2dStatField.put("2D_INTENSITY_50", "intensity_50"); + mtd2dStatField.put("2D_INTENSITY_75", "intensity_75"); + mtd2dStatField.put("2D_INTENSITY_90", "intensity_90"); + mtd2dStatField.put("2D_INTENSITY_N", "intensity_nn"); + } + + static { + mtd3dSingleStatField.put("3D_CENTROID_X", "centroid_x"); + mtd3dSingleStatField.put("3D_CENTROID_Y", "centroid_y"); + mtd3dSingleStatField.put("3D_CENTROID_T", "centroid_t"); + mtd3dSingleStatField.put("3D_CENTROID_LAT", "centroid_lat"); + mtd3dSingleStatField.put("3D_CENTROID_LON", "centroid_lon"); + mtd3dSingleStatField.put("3D_X_DOT", "x_dot"); + mtd3dSingleStatField.put("3D_Y_DOT", "y_dot"); + mtd3dSingleStatField.put("3D_AXIS_ANG", "axis_ang"); + mtd3dSingleStatField.put("3D_VOLUME", "volume"); + mtd3dSingleStatField.put("3D_START_TIME", "start_time"); + mtd3dSingleStatField.put("3D_END_TIME", "end_time"); + mtd3dSingleStatField.put("3D_DURATION", "end_time-start_time"); + mtd3dSingleStatField.put("3D_CDIST_TRAVELLED", "cdist_travelled"); + mtd3dSingleStatField.put("3D_INTENSITY_10", "intensity_10"); + mtd3dSingleStatField.put("3D_INTENSITY_25", "intensity_25"); + mtd3dSingleStatField.put("3D_INTENSITY_50", "intensity_50"); + mtd3dSingleStatField.put("3D_INTENSITY_75", "intensity_75"); + mtd3dSingleStatField.put("3D_INTENSITY_90", "intensity_90"); + mtd3dSingleStatField.put("3D_INTENSITY_N", "intensity_nn"); + } + + static { + mtd3dPairStatField.put("3D_SPACE_CENTROID_DIST", "space_centroid_dist"); + mtd3dPairStatField.put("3D_TIME_CENTROID_DELTA", "time_centroid_delta"); + mtd3dPairStatField.put("3D_AXIS_DIFF", "axis_diff"); + mtd3dPairStatField.put("3D_SPEED_DELTA", "speed_delta"); + mtd3dPairStatField.put("3D_DIRECTION_DIFF", "direction_diff"); + mtd3dPairStatField.put("3D_VOLUME_RATIO", "volume_ratio"); + mtd3dPairStatField.put("3D_START_TIME_DELTA", "start_time_delta"); + mtd3dPairStatField.put("3D_END_TIME_DELTA", "end_time_delta"); + mtd3dPairStatField.put("3D_INTERSECTION_VOLUME", "intersection_volume"); + mtd3dPairStatField.put("3D_DURATION_DIFF", "duration_diff"); + mtd3dPairStatField.put("3D_INTEREST", "interest"); + } + + static { + statsPerc.put("FCST_PERC", new String[]{}); + statsPerc.put("OBS_PERC", new String[]{}); + } + + + static { + calcStatCTC.add("BASER"); + calcStatCTC.add("BASER"); + calcStatCTC.add("ACC"); + calcStatCTC.add("FBIAS"); + calcStatCTC.add("PODY"); + calcStatCTC.add("POFD"); + calcStatCTC.add("PODN"); + calcStatCTC.add("FAR"); + calcStatCTC.add("CSI"); + calcStatCTC.add("GSS"); + } + + private MVUtil() { + throw new IllegalStateException("Utility class"); + } + + /** + * Build a list of strings representing consecutive dates between the input dates start and end, + * incrementing by incr number of seconds. It is assumed that the format of start and end is + * given by the java date format string format. The output dates will have the same format. + * + * @param start Beginning date, given in the format specified by the format input + * @param end End date, given in the format specified by the format input + * @param incr Number of seconds to increment between successive dates in the list + * @param format Java date format string, describing input and output dates + * @return List of date strings + */ + public static List buildDateList( + final String start, final String end, final int incr, + final String format) { + SimpleDateFormat formatDate = new SimpleDateFormat(format, Locale.US); + formatDate.setTimeZone(TimeZone.getTimeZone("UTC")); + List listDates = new ArrayList<>(); + + try { + Date dateStart = formatDate.parse(start); + Date dateEnd = formatDate.parse(end); + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + cal.setTime(dateStart); + + while ((incr > 0 && cal.getTime().getTime() <= dateEnd.getTime()) + || (incr < 0 && cal.getTime().getTime() >= dateEnd.getTime())) { + listDates.add(formatDate.format(cal.getTime())); + cal.add(Calendar.SECOND, incr); + } + + } catch (ParseException e) { + errorStream + .print(" ** ERROR: caught " + e.getClass() + " in buildDateList(): " + e.getMessage()); + + } + return listDates; + } + + /** + * Wrap the buildDateList implementation above, by parsing the specified node for start, end, incr + * and format and then returning the list of dates. + * + * @param node MVNode to parse for the date list parameters + * @return List of date strings + */ + public static List buildDateList(final MVNode node, final PrintStream printStream) + throws ValidationException { + String strStart = ""; + String strEnd = ""; + int intInc = 0; + String strFormat = ""; + + for (int j = 0; j < node.children.length; j++) { + MVNode nodeChild = node.children[j]; + if (nodeChild.tag.equals("inc")) { + intInc = Integer.parseInt(nodeChild.value); + } else if (nodeChild.tag.equals("format")) { + strFormat = nodeChild.value; } - }else { - String[] valuesArr = value.split(GROUP_SEPARATOR); - for (String v : valuesArr) { - newValues.add(v); + } + for (int j = 0; j < node.children.length; j++) { + MVNode nodeChild = node.children[j]; + if (nodeChild.tag.equals("start")) { + if (0 < nodeChild.children.length) { + strStart = parseDateOffset(nodeChild.children[0], strFormat); + } else { + strStart = nodeChild.value; + } + } else if (nodeChild.tag.equals("end")) { + strEnd = (0 < nodeChild.children.length + ? parseDateOffset(nodeChild.children[0], strFormat) : nodeChild.value); } - } + } + + return buildDateList(strStart, strEnd, intInc, strFormat); + } + + /** + * Build a String representation of the date specified by the input {@link MVNode}. + * The offset is taken either from the current date (default) or from the date specified by the + * input date. + * + * @param node MVNode structure specifying the offset + * @param format (optional) String representation of the input/output date formats + * @param date (optional) String representation of the date from which to offset + * @return String representation of the offset date + */ + public static String parseDateOffset(final MVNode node, final String format, final String date) { + int intOffset = 0; + int intHour = 0; + + for (int i = 0; i < node.children.length; i++) { + MVNode nodeChild = node.children[i]; + if (nodeChild.tag.equals("day_offset")) { + intOffset = Integer.parseInt(nodeChild.value); + } else if (nodeChild.tag.equals("hour")) { + intHour = Integer.parseInt(nodeChild.value); + } + } + + SimpleDateFormat formatOffset = new SimpleDateFormat(format, Locale.US); + formatOffset.setTimeZone(TimeZone.getTimeZone("UTC")); + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + try { + cal.setTime(formatOffset.parse(date)); + } catch (Exception e) { + } + cal.set(Calendar.HOUR_OF_DAY, intHour); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.add(Calendar.DATE, intOffset); + + return formatOffset.format(cal.getTime()); + } + + public static String parseDateOffset(final MVNode node, final String format) throws ValidationException { + return parseDateOffset(node, format, null); + } + + + /** + * Concatenate the elements of the input list with surrounding ticks and separated by commas for + * use in the where clause of a SQL query. For example, the function call + * buildValueList(new String[]{"a", "bb", "c"}) will return the string "'a', 'bb', + * 'c'". + * + * @param values The list of values to be concatenated + * @return The string of concatenated values for use in a SQL where clause + */ + public static String buildValueList(final String[] values) { + String[] localValues; + if (values != null && values.length > 0) { + List newValues = new ArrayList<>(); + for (String value : values) { + if (value.contains(GROUP_SEPARATOR)) { + boolean matchFound = patDateTime.matcher(value).find(); + if (matchFound) { + //this is a date - need to use patterns + Matcher m = patDateTime.matcher(value); + while (m.find()) { + newValues.add(m.group()); + } + } else { + String[] valuesArr = value.split(GROUP_SEPARATOR); + for (String v : valuesArr) { + newValues.add(v); + } + } + } else { + newValues.add(value); + } + } + localValues = newValues.toArray(new String[newValues.size()]); } else { - newValues.add(value); - } - } - localValues = newValues.toArray(new String[newValues.size()]); - } else { - localValues = values; - } - String strValueList = ""; - for (int i = 0; null != localValues && i < localValues.length; i++) { - if (0 < i) { - strValueList += ", " + "'" + localValues[i] + "'"; - } else { - strValueList += "" + "'" + localValues[i] + "'"; - } - } - return strValueList; - } - - /** - * Create a {@link MVDataTable} whose fields are the keys of the input table and whose rows - * represent every permutation of the values stored in the input table. It is assumed that the - * table contains a mapping from String to String[]. If the input table is an {@link - * MVOrderedMap}, the fields of the output MVDataTable are ordered in the same order as the keys - * of the input. - * - * @param table Contains key/value pairs of String/String[] which will be permuted - * @return MVDataTable whose rows are the permutations - */ - public static MVDataTable permute(final MVOrderedMap table) { - - if (null == table || 1 > table.size()) { - return new MVDataTable(); - } - - // use the ordered list of table entries, if appropriate - Map.Entry[] listVals; - listVals = table.getOrderedEntries(); - - // if the input table contains a single value, build and return the simplest table - if (1 == listVals.length) { - MVDataTable dtRet = new MVDataTable(); - String strField = (String) listVals[0].getKey(); - Object objVal = listVals[0].getValue(); - - // handle simple field value lists - if (objVal instanceof String[]) { - dtRet.addField(strField); - String[] listVal = (String[]) listVals[0].getValue(); - for (String val : listVal) { - MVOrderedMap tableRow = new MVOrderedMap(); - tableRow.put(strField, val); - dtRet.addRow(tableRow); - } - - // handle field value sets - } else if (objVal instanceof MVOrderedMap) { - dtRet.addField(strField + "_set"); - Map.Entry[] listValSet = ((MVOrderedMap) objVal).getOrderedEntries(); - for (int i = 0; i < listValSet.length; i++) { - MVOrderedMap tableRow = new MVOrderedMap(); - tableRow.put(strField + "_set", listValSet[i].getKey()); - dtRet.addRow(tableRow); - } - } - return dtRet; - } - - // if the input table contains more than one value, build the sub-table first - String strField = (String) listVals[0].getKey(); - MVOrderedMap tableSub = new MVOrderedMap(table); - tableSub.remove(strField); - MVDataTable dtSub = permute(tableSub); - - // build a new table with one copy of the sub-table for each value of the current field - MVOrderedMap[] listRows = dtSub.getRows(); - MVDataTable dtRet = new MVDataTable(dtSub.getFields()); - Object objVal = listVals[0].getValue(); - - // handle simple field value lists - if (objVal instanceof String[]) { - dtRet.addField(strField, "", 0); - String[] listVal = (String[]) listVals[0].getValue(); - for (int i = 0; i < listVal.length; i++) { - for (int j = 0; j < listRows.length; j++) { - MVOrderedMap tableRow = new MVOrderedMap(listRows[j]); - tableRow.put(strField, listVal[i]); - dtRet.addRow(tableRow); - } - } - - // handle field value sets - } else if (objVal instanceof MVOrderedMap) { - dtRet.addField(strField + "_set", "", 0); - Map.Entry[] listValSet = ((MVOrderedMap) objVal).getOrderedEntries(); - for (int i = 0; i < listValSet.length; i++) { - for (int j = 0; j < listRows.length; j++) { - MVOrderedMap tableRow = new MVOrderedMap(listRows[j]); - tableRow.put(strField + "_set", listValSet[i].getKey()); - dtRet.addRow(tableRow); - } - } - } - - return dtRet; - } - - - /** - * Sort the list of input thresholds, according to the numeric threshold value. - * - * @param thresh List of thresholds - * @return Sorted threshold list, by value - */ - public static List sortThresh(List thresh) { - thresh.sort( - new Comparator() { - private final Pattern PATTERN_WITH_FLOAT = Pattern.compile("(\\D*)([-+]?\\d*\\.?\\d+)"); - - public int compare(String s1, String s2) { - - // if the threshold contains a float like this '>.1' - - // add a missing 0 -> '>0.1' - if (s1.contains(">.") || s1.contains("<.") || s1.contains("=.") - || s1.contains("> .") || s1.contains("< .") || s1.contains("= .")) { - int pos = s1.indexOf('.'); - s1 = s1.substring(0, pos) + "0" + s1.substring(pos); + localValues = values; + } + String strValueList = ""; + for (int i = 0; null != localValues && i < localValues.length; i++) { + if (0 < i) { + strValueList += ", " + "'" + localValues[i] + "'"; + } else { + strValueList += "" + "'" + localValues[i] + "'"; + } + } + return strValueList; + } + + /** + * Create a {@link MVDataTable} whose fields are the keys of the input table and whose rows + * represent every permutation of the values stored in the input table. It is assumed that the + * table contains a mapping from String to String[]. If the input table is an {@link + * MVOrderedMap}, the fields of the output MVDataTable are ordered in the same order as the keys + * of the input. + * + * @param table Contains key/value pairs of String/String[] which will be permuted + * @return MVDataTable whose rows are the permutations + */ + public static MVDataTable permute(final MVOrderedMap table) { + + if (null == table || 1 > table.size()) { + return new MVDataTable(); + } + + // use the ordered list of table entries, if appropriate + Map.Entry[] listVals; + listVals = table.getOrderedEntries(); + + // if the input table contains a single value, build and return the simplest table + if (1 == listVals.length) { + MVDataTable dtRet = new MVDataTable(); + String strField = (String) listVals[0].getKey(); + Object objVal = listVals[0].getValue(); + + // handle simple field value lists + if (objVal instanceof String[]) { + dtRet.addField(strField); + String[] listVal = (String[]) listVals[0].getValue(); + for (String val : listVal) { + MVOrderedMap tableRow = new MVOrderedMap(); + tableRow.put(strField, val); + dtRet.addRow(tableRow); } - if (s2.contains(">.") || s2.contains("<.") || s2.contains("=.") - || s2.contains("> .") || s2.contains("< .") || s2.contains("= .")) { - int pos = s2.indexOf('.'); - s2 = s2.substring(0, pos) + "0" + s2.substring(pos); + + // handle field value sets + } else if (objVal instanceof MVOrderedMap) { + dtRet.addField(strField + "_set"); + Map.Entry[] listValSet = ((MVOrderedMap) objVal).getOrderedEntries(); + for (int i = 0; i < listValSet.length; i++) { + MVOrderedMap tableRow = new MVOrderedMap(); + tableRow.put(strField + "_set", listValSet[i].getKey()); + dtRet.addRow(tableRow); } - Matcher m1 = PATTERN_WITH_FLOAT.matcher(s1); - Matcher m2 = PATTERN_WITH_FLOAT.matcher(s2); - - // The only way find() could fail is at the end of a string - while (m1.find() && m2.find()) { - - - // matcher.group(2) fetches any digits captured by the - // second parentheses in PATTERN. - if (m1.group(2).isEmpty()) { - return m2.group(2).isEmpty() ? 0 : -1; - } else if (m2.group(2).isEmpty()) { - return +1; - } - - Float n1 = Float.valueOf(m1.group(2)); - Float n2 = Float.valueOf(m2.group(2)); - int numberCompare = n1.compareTo(n2); - if (0 != numberCompare) { - return numberCompare; - } - // matcher.group(1) fetches any non-digits captured by the - // first parentheses in PATTERN. - int nonDigitCompare = m1.group(1).compareTo(m2.group(1)); - if (0 != nonDigitCompare) { - return nonDigitCompare; - } + } + return dtRet; + } + + // if the input table contains more than one value, build the sub-table first + String strField = (String) listVals[0].getKey(); + MVOrderedMap tableSub = new MVOrderedMap(table); + tableSub.remove(strField); + MVDataTable dtSub = permute(tableSub); + + // build a new table with one copy of the sub-table for each value of the current field + MVOrderedMap[] listRows = dtSub.getRows(); + MVDataTable dtRet = new MVDataTable(dtSub.getFields()); + Object objVal = listVals[0].getValue(); + + // handle simple field value lists + if (objVal instanceof String[]) { + dtRet.addField(strField, "", 0); + String[] listVal = (String[]) listVals[0].getValue(); + for (int i = 0; i < listVal.length; i++) { + for (int j = 0; j < listRows.length; j++) { + MVOrderedMap tableRow = new MVOrderedMap(listRows[j]); + tableRow.put(strField, listVal[i]); + dtRet.addRow(tableRow); } + } - // Handle if one string is a prefix of the other. - // Nothing comes before something. - if (m1.hitEnd() && m2.hitEnd()) { - return 1; + // handle field value sets + } else if (objVal instanceof MVOrderedMap) { + dtRet.addField(strField + "_set", "", 0); + Map.Entry[] listValSet = ((MVOrderedMap) objVal).getOrderedEntries(); + for (int i = 0; i < listValSet.length; i++) { + for (int j = 0; j < listRows.length; j++) { + MVOrderedMap tableRow = new MVOrderedMap(listRows[j]); + tableRow.put(strField + "_set", listValSet[i].getKey()); + dtRet.addRow(tableRow); } - if (m1.hitEnd()) { - return -1; + } + } + + return dtRet; + } + + + /** + * Sort the list of input thresholds, according to the numeric threshold value. + * + * @param thresh List of thresholds + * @return Sorted threshold list, by value + */ + public static List sortThresh(List thresh) { + thresh.sort( + new Comparator() { + private final Pattern PATTERN_WITH_FLOAT = Pattern.compile("(\\D*)([-+]?\\d*\\.?\\d+)"); + + public int compare(String s1, String s2) { + + // if the threshold contains a float like this '>.1' - + // add a missing 0 -> '>0.1' + if (s1.contains(">.") || s1.contains("<.") || s1.contains("=.") + || s1.contains("> .") || s1.contains("< .") || s1.contains("= .")) { + int pos = s1.indexOf('.'); + s1 = s1.substring(0, pos) + "0" + s1.substring(pos); + } + if (s2.contains(">.") || s2.contains("<.") || s2.contains("=.") + || s2.contains("> .") || s2.contains("< .") || s2.contains("= .")) { + int pos = s2.indexOf('.'); + s2 = s2.substring(0, pos) + "0" + s2.substring(pos); + } + Matcher m1 = PATTERN_WITH_FLOAT.matcher(s1); + Matcher m2 = PATTERN_WITH_FLOAT.matcher(s2); + + // The only way find() could fail is at the end of a string + while (m1.find() && m2.find()) { + + + // matcher.group(2) fetches any digits captured by the + // second parentheses in PATTERN. + if (m1.group(2).isEmpty()) { + return m2.group(2).isEmpty() ? 0 : -1; + } else if (m2.group(2).isEmpty()) { + return +1; + } + + Float n1 = Float.valueOf(m1.group(2)); + Float n2 = Float.valueOf(m2.group(2)); + int numberCompare = n1.compareTo(n2); + if (0 != numberCompare) { + return numberCompare; + } + // matcher.group(1) fetches any non-digits captured by the + // first parentheses in PATTERN. + int nonDigitCompare = m1.group(1).compareTo(m2.group(1)); + if (0 != nonDigitCompare) { + return nonDigitCompare; + } + } + + // Handle if one string is a prefix of the other. + // Nothing comes before something. + if (m1.hitEnd() && m2.hitEnd()) { + return 1; + } + if (m1.hitEnd()) { + return -1; + } + if (m2.hitEnd()) { + return 1; + } + return +1; + } + } + ); + + return thresh; + } + + + /** + * Sort the list of input levels, according to the first numeric level value. + * + * @param lev List of thresholds + * @return Sorted threshold list, by value + */ + public static List sortLev(final List lev) { + return sortVals(lev, MVUtil.lev); + } + + /** + * Sort the list of Interp Pnts, according to the first numeric level value. + * + * @param lev List of Interp Pnts + * @return Sorted Inter Pnts list, by value + */ + public static List sortInterpPnts(final List lev) throws ValidationException { + List resultInt = new ArrayList<>(lev.size()); + for (String interpPnt : lev) { + try { + resultInt.add(Integer.valueOf(interpPnt)); + } catch (NumberFormatException e) { + throw new ValidationException("interp_pnt is invalid"); + } + } + Collections.sort(resultInt); + List resultStr = new ArrayList<>(resultInt.size()); + for (Integer interpPnt : resultInt) { + resultStr.add(String.valueOf(interpPnt)); + } + return resultStr; + } + + /** + * Sort the input list of values by parsing them with the input pattern and sort them according to + * the numerical portion (assumed to be group 2 of the matched pattern). + * + * @param vals List of String representations of the values + * @param pat Pattern used to parse the input values + * @return Sorted list, by numerical value + */ + public static List sortVals( + final List vals, + final Pattern pat) { + + // parse the input values and store the numerical values in a sortable array + double[] listVal = new double[vals.size()]; + Map tableVal = new HashMap<>(); + double dblInvalid = -.00001; + for (int i = 0; i < vals.size(); i++) { + String val = vals.get(i); + + // apply the pattern to the value + double dblVal; + //if value is double and ends with '.' - remove '.' to match the pattern + if (val.endsWith(".")) { + val = val.substring(0, val.length() - 1); + } + Matcher mat = pat.matcher(val); + + // if the value matches, parse out the numerical value + if (mat.matches()) { + if (mat.groupCount() == 0) { + dblVal = Double.parseDouble(val); + } else { + dblVal = Double.parseDouble(mat.group(2)); + if (3 == mat.groupCount() && null != mat.group(3)) { + dblVal = (dblVal + Double.parseDouble(mat.group(3))) / 2; + } } - if (m2.hitEnd()) { - return 1; + } + + // otherwise, use the literal value with a default numerical value + else { + dblVal = dblInvalid; + dblInvalid -= .00001; + } + + // verify and store the numerical value and the value pair + listVal[i] = dblVal; + Double dblKey = listVal[i]; + Object objVal = vals.get(i); + if (tableVal.containsKey(dblKey)) { + Object objValCur = tableVal.get(dblKey); + ArrayList listValCur = new ArrayList(); + if (objValCur instanceof String) { + listValCur.add(objValCur); + listValCur.add(val); + } else { + ((ArrayList) objValCur).add(val); } - return +1; - } + objVal = listValCur; } - ); - - return thresh; - } - - - /** - * Sort the list of input levels, according to the first numeric level value. - * - * @param lev List of thresholds - * @return Sorted threshold list, by value - */ - public static List sortLev(final List lev) { - return sortVals(lev, MVUtil.lev); - } - - /** - * Sort the list of Interp Pnts, according to the first numeric level value. - * - * @param lev List of Interp Pnts - * @return Sorted Inter Pnts list, by value - */ - public static List sortInterpPnts(final List lev) throws ValidationException { - List resultInt = new ArrayList<>(lev.size()); - for (String interpPnt : lev) { - try { - resultInt.add(Integer.valueOf(interpPnt)); - } catch (NumberFormatException e) { - throw new ValidationException("interp_pnt is invalid"); - } - } - Collections.sort(resultInt); - List resultStr = new ArrayList<>(resultInt.size()); - for (Integer interpPnt : resultInt) { - resultStr.add(String.valueOf(interpPnt)); - } - return resultStr; - } - - /** - * Sort the input list of values by parsing them with the input pattern and sort them according to - * the numerical portion (assumed to be group 2 of the matched pattern). - * - * @param vals List of String representations of the values - * @param pat Pattern used to parse the input values - * @return Sorted list, by numerical value - */ - public static List sortVals( - final List vals, - final Pattern pat) { - - // parse the input values and store the numerical values in a sortable array - double[] listVal = new double[vals.size()]; - Map tableVal = new HashMap<>(); - double dblInvalid = -.00001; - for (int i = 0; i < vals.size(); i++) { - String val = vals.get(i); - - // apply the pattern to the value - double dblVal; - //if value is double and ends with '.' - remove '.' to match the pattern - if (val.endsWith(".")) { - val = val.substring(0, val.length() - 1); - } - Matcher mat = pat.matcher(val); - - // if the value matches, parse out the numerical value - if (mat.matches()) { - if (mat.groupCount() == 0) { - dblVal = Double.parseDouble(val); - } else { - dblVal = Double.parseDouble(mat.group(2)); - if (3 == mat.groupCount() && null != mat.group(3)) { - dblVal = (dblVal + Double.parseDouble(mat.group(3))) / 2; - } - } - } - - // otherwise, use the literal value with a default numerical value - else { - dblVal = dblInvalid; - dblInvalid -= .00001; - } - - // verify and store the numerical value and the value pair - listVal[i] = dblVal; - Double dblKey = listVal[i]; - Object objVal = vals.get(i); - if (tableVal.containsKey(dblKey)) { - Object objValCur = tableVal.get(dblKey); - ArrayList listValCur = new ArrayList(); - if (objValCur instanceof String) { - listValCur.add(objValCur); - listValCur.add(val); - } else { - ((ArrayList) objValCur).add(val); + tableVal.put(listVal[i], objVal); } - objVal = listValCur; - } - tableVal.put(listVal[i], objVal); - } - - // sort the numerical values and build a sorted list of values - Arrays.sort(listVal); - ArrayList listRet = new ArrayList(); - Map tableAdded = new HashMap<>(); - for (int i = 0; i < listVal.length; i++) { - - // verify that the values have not already been added - Double dblKey = listVal[i]; - if (tableAdded.containsKey(dblKey)) { - continue; - } - // if not, add the value(s) to the return list - Object objValCur = tableVal.get(dblKey); - if (objValCur instanceof String) { - listRet.add(listRet.size(), objValCur); - } else { - ArrayList listValCur = (ArrayList) objValCur; - for (Object valCur : listValCur) { - listRet.add(listRet.size(), valCur); - } - } - - tableAdded.put(dblKey, "true"); - } - - return listRet; - } - - /** - * Parse, format and sort the input list of lead times, removing the trailing 0000, if requested. - * - * @param lead List of lead time values - * @return Sorted list of formatted lead times, by numerical value - */ - public static List sortFormatLead( - final List lead) { - - // parse and format the leads and store the numerical values in a sortable array - double[] listVal = new double[lead.size()]; - Map tableVal = new HashMap<>(); - for (int i = 0; i < lead.size(); i++) { - listVal[i] = Double.parseDouble(lead.get(i)); - tableVal.put(listVal[i], lead.get(i)); - } - - // sort the lead numerical values and build a sorted list of leads - Arrays.sort(listVal); - List listRet = new ArrayList<>(lead.size()); - for (int i = 0; i < listVal.length; i++) { - listRet.add(i, tableVal.get(listVal[i])); - } - - return listRet; - } - - /** - * removes the trailing .0 from dates . - * - * @return list of formatted dates - */ - public static List formatDates(final List dates) { - List listRet = new ArrayList<>(dates.size()); - for (int i = 0; i < dates.size(); i++) { - listRet.add(i, dates.get(i).replace(".0", "")); - } - return listRet; - } - - public static List sortHour(final List hour) throws ValidationException { - - List hoursInt = new ArrayList<>(); - for (String hourStr : hour) { - try { - hoursInt.add(Integer.valueOf(hourStr)); - } catch (NumberFormatException e) { - throw new ValidationException("the hour " + hourStr + " is invalid"); - } - } - Collections.sort(hoursInt); - - List result = new ArrayList<>(); - for (Integer hourInt : hoursInt) { - result.add(String.format("%02d", hourInt)); - } - return result; - - } - - - /** - * Pads input str with spaces appended to the end so that the length of the returned String is at - * least width characters - * - * @param str The string to pad - * @param width The minimum number of characters in the returned String - * @return the padded version of the input str - */ - public static String padEnd(String str, final String pad, final int width) { - while (width > str.length()) { - str += pad; - } - return str; - } - - public static String padEnd(final String str, final int width) { - return padEnd(str, " ", width); - } - - - /** - * Pads input str with spaces appended to the beginning so that the length of the returned String - * is at least width characters - * - * @param str The string to pad - * @param width The minimum number of characters in the returned String - * @return the padded version of the input str - */ - public static String padBegin(String str, final String pad, final int width) { - while (width > str.length()) { - str = pad + str; - } - return str; - } - - public static String padBegin(final String str, final int width) { - return padBegin(str, " ", width); - } - - public static String padBegin(final String str) { - return padBegin(str, 16); - } - - /** - * Create a string representation for the input time span, which should represent milliseconds - * between events. For example, a time span message can be generated as follows: - * formatTimeStamp(dateEnd.getTime() - dateStart.getTime()) - * - * @param span Time span, in milliseconds - * @return Time span in format [days]d H:mm:ss.mmmm - */ - public static String formatTimeSpan(long span) { - long intDay = span / (24L * 60L * 60L * 1000L); - span -= intDay * 24L * 60L * 60L * 1000L; - long intHr = span / (60L * 60L * 1000L); - span -= intHr * 60L * 60L * 1000L; - long intMin = span / (60L * 1000L); - span -= intMin * 60L * 1000L; - long intSec = span / 1000L; - span -= intSec * 1000L; - long intMs = span; - - return (0 < intDay ? Long.toString(intDay) + "d " : "") + Long.toString(intHr) - + (10 > intMin ? ":0" : ":") + Long.toString(intMin) - + (10 > intSec ? ":0" : ":") + Long.toString(intSec) + "." - + (100 > intMs ? "0" + (10 > intMs ? "0" : "") : "") + Long.toString(intMs); - } - - - /** - * Append the first array with the values of the second - * - * @param l1 Array to be appended - * @param l2 Array to append - * @return The combined array - */ - public static Object[] append(final Object[] l1, final Object[] l2, final Object[] cast) { - List listRet = Arrays.asList(l1); - listRet.addAll(Arrays.asList(l2)); - return listRet.toArray(cast); - - } - - public static String[] append(final String[] s1, final String[] s2) { - return (String[]) append(s1, s2, new String[]{}); - } - - public static String[] append(final String s1, final String[] s2) { - return append(new String[]{s1}, s2); - } - - public static String[] append(final String[] s1, final String s2) { - return append(s1, new String[]{s2}); - } - - - public static Map.Entry[] append(final Map.Entry[] s1, final Map.Entry[] s2) { - return (Map.Entry[]) append(s1, s2, new Map.Entry[]{}); - } - - public static MVPlotJob[] append(final MVPlotJob[] s1, final MVPlotJob[] s2) { - return (MVPlotJob[]) append(s1, s2, new MVPlotJob[]{}); - } - - public static int[] append(int[] s1, int[] s2) { - if (null == s1) { - s1 = new int[]{}; - } - if (null == s2) { - s2 = new int[]{}; - } - int r = 0; - int[] ret = new int[s1.length + s2.length]; - for (int i = 0; i < s1.length; i++) { - ret[r++] = s1[i]; - } - for (int i = 0; i < s2.length; i++) { - ret[r++] = s2[i]; - } - return ret; - } - - public static String[] unique(final String[] dataArr) { - Map table = new HashMap<>(); - for (String data : dataArr) { - if (!table.containsKey(data)) { - table.put(data, "true"); - } - } - Set strings = table.keySet(); - return strings.toArray(new String[strings.size()]); - } - - public static int sum(int[] dataArr) { - if (1 > dataArr.length) { - return 0; - } - int intSum = 0; - for (int data : dataArr) { - intSum += data; - } - return intSum; - } - - /** - * Attempt to convert the input ArrayList, which is assumed to contain all Strings, to a - * String[]. - * - * @param list ArrayList to convert - * @return Converted list - */ - public static String[] toArray(final List list) { - return list.toArray(new String[list.size()]); - } - - public static List toArrayList(final String[] list) { - List ret = new ArrayList<>(); - ret.addAll(Arrays.asList(list)); - return ret; - } - - /** - * Create a deep copy of the input list - * - * @param list List to copy - * @return Copied list - */ - public static MVOrderedMap[] copyList(final MVOrderedMap[] list) { - MVOrderedMap[] listRet = new MVOrderedMap[list.length]; - for (int i = 0; i < list.length; i++) { - listRet[i] = new MVOrderedMap(list[i]); - } - return listRet; - } - - /** - * Create a deep copy of the input list - * - * @param list List to copy - * @return Copied list - */ - public static String[] copyList(final String[] list) { - return list.clone(); - } - - - /** - * Returns a string representation of the MVOrderedMap in R declaration syntax - * - * @param map Data structure to convert to R list representation - * @return The R-syntax representation of the input map - */ - public static String getRDecl(final MVOrderedMap map) { - String strRDecl = "list(\n"; - List keys = map.getListKeys(); - String[] listKeys = (String[]) keys.toArray(new String[keys.size()]); - for (int i = 0; i < listKeys.length; i++) { - if (0 < i) { - strRDecl += ",\n" + MVUtil.padBegin("`" + listKeys[i].replace("&", "&").replace(">", ">") - .replace("<", "<") + "`") + " = "; - } else { - strRDecl += "" + MVUtil.padBegin("`" + listKeys[i].replace("&", "&").replace(">", ">") - .replace("<", "<") + "`") + " = "; - } - Object objVal = map.get(listKeys[i]); - if (objVal instanceof String) { - strRDecl += "\"" + objVal.toString() + "\""; - } else if (objVal instanceof String[]) { - strRDecl += "c("; - String[] listVal = (String[]) objVal; - for (int j = 0; j < listVal.length; j++) { - strRDecl += (0 < j ? ", " : "") + "\"" + listVal[j] + "\""; - } - strRDecl += ")"; - } else if (objVal instanceof MVOrderedMap) { - strRDecl += ((MVOrderedMap) objVal).getRDecl(); - } else { - strRDecl += "\"???\",\n"; - } - } - strRDecl += "\n)"; - return strRDecl; - } - - public static Map getYamlDecl(final MVOrderedMap map) { - Map result = new LinkedHashMap<>(); - List keys = map.getListKeys(); - for (String key : keys) { - Object objVal = map.get(key); - if (objVal instanceof String) { - result.put(key, new String[]{((String) objVal).replace("&", "&").replace(">", ">") - .replace("<", "<")}); - } else if (objVal instanceof String[]) { - result.put(key, objVal); - } else if (objVal instanceof MVOrderedMap) { - result.put(key, ((MVOrderedMap) objVal).getYamlDecl()); - } - } - return result; - } - - - public static String[] parseModeStat(final String stat) { - Matcher mat = _patModeStat.matcher(stat); - if (!mat.matches()) { - return new String[]{stat}; - } - return new String[]{mat.group(1), mat.group(2)}; - } - - - /** - * Determine the database line_data table in which the input statistic is stored. For mode stats, - * consider only the first portion of the stat name. If the stat is not found in any table, - * return an empty string. - * - * @param strStat stat name to look up - * @return the name of the database line_data table which contains the stat - */ - public static String getStatTable(final String strStat) { - String strStatMode = parseModeStat(strStat)[0]; - String strStatMrd = ""; - if (strStatMode.equals(strStat)) { - String[] listStatComp = strStat.split("_"); - strStatMrd = strStat.replace("_" + listStatComp[listStatComp.length - 1], ""); - } - - //TODO mtdRatioField and modeRatioField contain the same fields - - if (statsCnt.containsKey(strStat)) { - return "line_data_cnt"; - } else if (statsCts.containsKey(strStat)) { - return "line_data_cts"; - } else if (statsNbrcnt.containsKey(strStat)) { - return "line_data_nbrcnt"; - } else if (statsNbrcts.containsKey(strStat)) { - return "line_data_nbrcts"; - } else if (statsPstd.containsKey(strStat)) { - return "line_data_pstd"; - } else if (statsMcts.containsKey(strStat)) { - return "line_data_mcts"; - } else if (statsRhist.containsKey(strStat)) { - return "line_data_rhist"; - } else if (statsVl1l2.containsKey(strStat)) { - return "line_data_vl1l2"; - } else if (statsVal1l2.containsKey(strStat)) { - return "line_data_val1l2"; - } else if (modeSingleStatField.containsKey(strStatMode)) { - return "mode_obj_single"; - } else if (modePairStatField.containsKey(strStatMode)) { - return "mode_obj_pair"; - } else if (modeRatioField.contains(strStat)) { - return "mode_obj_single"; - } else if (statsEnscnt.containsKey(strStat)) { - return "line_data_enscnt"; - } else if (statsMpr.containsKey(strStat)) { - return "line_data_mpr"; - } else if (statsOrank.containsKey(strStat)) { - return "line_data_orank"; - } else if (statsSsvar.containsKey(strStat)) { - return "line_data_ssvar"; - } else if (mtd3dSingleStatField.containsKey(strStatMrd)) { - return "mtd_3d_obj_single"; - } else if (mtd3dPairStatField.containsKey(strStatMrd)) { - return "mtd_3d_obj_pair"; - } else if (mtd2dStatField.containsKey(strStatMrd)) { - return "mtd_2d_obj"; - } else if (mtdRatioField.contains(strStat)) { - return "mtd_3d_obj_single"; - } else if (statsVcnt.containsKey(strStat)) { - return "line_data_vcnt"; - } else if (statsEcnt.containsKey(strStat)) { - return "line_data_ecnt"; - } else if (statsPerc.containsKey(strStat)) { - return "line_data_perc"; - } else if (statsDmap.containsKey(strStat)) { - return "line_data_dmap"; - } else if (statsRps.containsKey(strStat)) { - return "line_data_rps"; - } else if (statsCtc.containsKey(strStat)) { - return "line_data_ctc"; - } else if (statsNbrctc.containsKey(strStat)) { - return "line_data_ctc"; - } else if (statsPct.containsKey(strStat)) { - return "line_data_pct"; - } else if (statsSl1l2.containsKey(strStat)) { - return "line_data_sl1l2"; - } else if (statsSal1l2.containsKey(strStat)) { - return "line_data_sal1l2"; - } else if (statsGrad.containsKey(strStat)) { - return "line_data_grad"; - } else if (statsSsidx.containsKey(strStat)) { - return "line_data_ssidx"; - } else if (statsSeeps.containsKey(strStat)) { - return "line_data_seeps"; - } else { - return ""; - } - } - - - public static MvResponse runRscript( - final String rscript, - final String script) { - return runRscript(rscript, script, new String[]{}, null); - } - - public static MvResponse runRscript( - final String rscriptCommand, final String scriptName, - final String[] args) { - return runRscript(rscriptCommand, scriptName, args, null); - } - - /** - * Run the input R script named r using the Rscript command. The output and error output will be - * written to standard output. - * - * @param rscriptCommand Rscript command - * @param scriptName R script to run - * @param args (optional) Arguments to pass to the R script - * @throws Exception - */ - public static MvResponse runRscript( - final String rscriptCommand, final String scriptName, - final String[] args, final String[] env) { - - MvResponse mvResponse = new MvResponse(); - - // build a list of arguments - StringBuilder argList = new StringBuilder(); - for (int i = 0; null != args && i < args.length; i++) { - argList.append(' ').append(args[i]); - } - - Process proc = null; - InputStreamReader inputStreamReader = null; - InputStreamReader errorInputStreamReader = null; - - BoundedBufferedReader readerProcStd = null; - BoundedBufferedReader readerProcErr = null; - - boolean boolExit = false; - int intExitStatus = 0; - StringBuilder strProcStd = new StringBuilder(); - StringBuilder strProcErr = new StringBuilder(); - - - try { - String rscriptCommandClean = cleanString(rscriptCommand); - String scriptNameClean = cleanString(scriptName); - String strArgListClean = cleanString(argList.toString()); - - - proc = Runtime.getRuntime() - .exec(rscriptCommandClean + " " + scriptNameClean + strArgListClean, - env, - new File(System.getProperty("user.home"))); - inputStreamReader = new InputStreamReader(proc.getInputStream()); - errorInputStreamReader = new InputStreamReader(proc.getErrorStream()); - - readerProcStd = new BoundedBufferedReader(inputStreamReader, 50000, 2048); - readerProcErr = new BoundedBufferedReader(errorInputStreamReader, 50000, 2048); - while (!boolExit) { - try { - intExitStatus = proc.exitValue(); - boolExit = true; - } catch (IllegalThreadStateException e) { - logger.debug(e.getMessage()); + // sort the numerical values and build a sorted list of values + Arrays.sort(listVal); + ArrayList listRet = new ArrayList(); + Map tableAdded = new HashMap<>(); + for (int i = 0; i < listVal.length; i++) { + + // verify that the values have not already been added + Double dblKey = listVal[i]; + if (tableAdded.containsKey(dblKey)) { + continue; + } + + // if not, add the value(s) to the return list + Object objValCur = tableVal.get(dblKey); + if (objValCur instanceof String) { + listRet.add(listRet.size(), objValCur); + } else { + ArrayList listValCur = (ArrayList) objValCur; + for (Object valCur : listValCur) { + listRet.add(listRet.size(), valCur); + } + } + + tableAdded.put(dblKey, "true"); } - while (readerProcStd.ready()) { - String line = readerProcStd.readLineBounded(); - strProcStd.append(line).append('\n'); + return listRet; + } + + /** + * Parse, format and sort the input list of lead times, removing the trailing 0000, if requested. + * + * @param lead List of lead time values + * @return Sorted list of formatted lead times, by numerical value + */ + public static List sortFormatLead( + final List lead) { + + // parse and format the leads and store the numerical values in a sortable array + double[] listVal = new double[lead.size()]; + Map tableVal = new HashMap<>(); + for (int i = 0; i < lead.size(); i++) { + listVal[i] = Double.parseDouble(lead.get(i)); + tableVal.put(listVal[i], lead.get(i)); } - while (readerProcErr.ready()) { - String line = readerProcErr.readLineBounded(); - strProcErr.append(line).append('\n'); + + // sort the lead numerical values and build a sorted list of leads + Arrays.sort(listVal); + List listRet = new ArrayList<>(lead.size()); + for (int i = 0; i < listVal.length; i++) { + listRet.add(i, tableVal.get(listVal[i])); } - } - } catch (SecurityException | IOException | IllegalArgumentException e) { - logger.error( e.getMessage()); - } finally { - if (inputStreamReader != null) { - try { - inputStreamReader.close(); - } catch (IOException e) { - logger.error( e.getMessage()); + return listRet; + } + + /** + * removes the trailing .0 from dates . + * + * @return list of formatted dates + */ + public static List formatDates(final List dates) { + List listRet = new ArrayList<>(dates.size()); + for (int i = 0; i < dates.size(); i++) { + listRet.add(i, dates.get(i).replace(".0", "")); } - } - if (errorInputStreamReader != null) { - try { - errorInputStreamReader.close(); - } catch (IOException e) { - logger.error( e.getMessage()); + return listRet; + } + + public static List sortHour(final List hour) throws ValidationException { + + List hoursInt = new ArrayList<>(); + for (String hourStr : hour) { + try { + hoursInt.add(Integer.valueOf(hourStr)); + } catch (NumberFormatException e) { + throw new ValidationException("the hour " + hourStr + " is invalid"); + } } - } - if (readerProcStd != null) { - try { - readerProcStd.close(); - } catch (IOException e) { - logger.error( e.getMessage()); + Collections.sort(hoursInt); + + List result = new ArrayList<>(); + for (Integer hourInt : hoursInt) { + result.add(String.format("%02d", hourInt)); } - } - if (readerProcErr != null) { - try { - readerProcErr.close(); - } catch (IOException e) { - logger.error( e.getMessage()); - } - } - if (proc != null) { - proc.destroy(); - } - - } - - String type = "Rscript"; - if (rscriptCommand.contains("python")) { - type = PYTHON; - } - if (strProcStd.length() > 0) { - mvResponse.setInfoMessage( - "\n==== Start " + type + " output ====\n" + strProcStd + "==== End " + type + " output ====\n"); - } - // add errors if they exist - only the last line - if (strProcErr.length() > 0) { - String[] errorLines = strProcErr.toString().split("\\r?\\n|\\r"); - mvResponse.setInfoMessage( - mvResponse.getInfoMessage() + "\n==== Start " + type + " ERROR ====\n" + errorLines[errorLines.length-1] + "\n==== End " + type + " error ====\n" - - ); - - mvResponse.setErrorMessage( - "==== Start " + type + " error ====\n" + strProcErr + "==== End " + type + " ERROR ===="); - } - mvResponse.setSuccess(0 == intExitStatus); - return mvResponse; - } - - /** - * Populate the template tags in the input template file named tmpl with values from the input - * table vals and write the result to the output file named output. - * - * @param tmpl Template file to populate - * @param output Output file to write - * @param vals Table containing values corresponding to tags in the input template - * @throws Exception - */ - public static void populateTemplateFile( - final String tmpl, final String output, - final Map vals) throws IOException { - try (FileReader fileReader = new FileReader(tmpl); - BoundedBufferedReader reader = new BoundedBufferedReader(fileReader); - PrintStream writer = new PrintStream(output)) { - while (reader.ready()) { - String strTmplLine = reader.readLineBounded(); - String strOutputLine = strTmplLine; - - Matcher matRtmplLine = patRTmpl.matcher(strTmplLine); - while (matRtmplLine.find()) { - String strRtmplTag = matRtmplLine.group(1); - if (!vals.containsKey(strRtmplTag)) { - continue; - } - String strRTagVal = (String) vals.get(strRtmplTag); - if (strRTagVal != null) { - strOutputLine = strOutputLine.replace("#<" + strRtmplTag + ">#", strRTagVal); - } - - } - - writer.println(strOutputLine); - } - } catch (IOException e) { - logger.error( e.getMessage()); - throw e; - } - - } - - - /** - * Build a list of the fcst_var/stat combinations stored in the input structure. The output - * list is structured as a list of pairs of Strings, with the first element storing the fcst_var - * and the second element storing the associated statistic. - * - * @param mapDep or structure from a MVPlotJob - * @return a list of fcst_var/stat pairs - */ - public static String[][] buildFcstVarStatList(final MVOrderedMap mapDep) { - List listRet = new ArrayList(); - String[] listFcstVar = mapDep.getKeyList(); - String[] listStat; - for (int intFcstVar = 0; intFcstVar < listFcstVar.length; intFcstVar++) { - listStat = (String[]) mapDep.get(listFcstVar[intFcstVar]); - for (int intStat = 0; intStat < listStat.length; intStat++) { - listRet.add(new String[]{listFcstVar[intFcstVar], listStat[intStat]}); - } - } - return (String[][]) listRet.toArray(new String[][]{}); - } - - - /** - * Construct the template map for the specified permutation of plot_fix values, using the - * specified set values. - * - * @param mapPlotFix plot_fix field/value pairs to use in populating the template values - * @param mapPlotFixVal values used for sets - * @throws Exception - */ - public static Map.Entry[] buildPlotFixTmplMap( - final MVOrderedMap mapPlotFix, - final MVOrderedMap mapPlotFixVal) { - Map.Entry[] listPlotFixVal = mapPlotFix.getOrderedEntries(); - // replace fixed value set names with their value maps - ArrayList listPlotFixValAdj = new ArrayList(); - for (Map.Entry aListPlotFixVal : listPlotFixVal) { - String strFixVar = aListPlotFixVal.getKey().toString(); - if (!strFixVar.endsWith("_set")) { - listPlotFixValAdj.add(aListPlotFixVal); - continue; - } - - String strFixVarAdj = strFixVar.replaceAll("_set$", ""); - MVOrderedMap mapFixSet = (MVOrderedMap) mapPlotFixVal.get(strFixVarAdj); - listPlotFixValAdj.add(new MVMapEntry(strFixVarAdj, mapFixSet)); - } - listPlotFixVal = (Map.Entry[]) listPlotFixValAdj - .toArray(new Map.Entry[listPlotFixValAdj.size()]); - - return listPlotFixVal; - } - - /** - * check if the aggregation type is compatible with the statistic We're currently only aggregating - * SL1L2 -> CNT and CTC -> CTS - * - * @param tableStats - * @param strStat - * @param aggType - * @throws Exception - */ - public static void isAggTypeValid( - final Map tableStats, final String strStat, - final String aggType) throws ValidationException { - //check if aggType is allowed for this stat - String[] types = tableStats.get(strStat); - boolean isFound = false; - for (String type : types) { - if (type.equals(aggType)) { - isFound = true; - break; - } - } - if (!isFound) { - throw new ValidationException( - "aggregation type " + aggType + " isn't compatible with the statistic " + strStat); - } - } - - public static String findValue( - final String[] listToken, final List headerNames, - final String header) { - int pos = headerNames.indexOf(header); - if (pos >= 0 && pos < listToken.length) { - return listToken[pos]; - } else { - return "NA"; - } - } - - - public static void updateLog4jConfiguration() { - - try { - String jarPath = MVUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI() - .getPath(); - URI imgurl = new URI("jar:file:" + jarPath + "!/edu/ucar/metviewer/resources/log4j2.xml"); - Logger l = (Logger) LogManager.getLogger(LogManager.ROOT_LOGGER_NAME); - l.getContext().setConfigLocation(imgurl); - } catch (URISyntaxException | SecurityException | UnsupportedOperationException e) { - logger.error( e.getMessage()); - } - - } - - /** - * Format the input String so that it conforms to R variable name standards - * - * @param in String to format - * @return Formatted String - */ - public static String replaceSpecialChars(final String in) { - - String strFormatR = in; - Matcher matProb = MVUtil.prob.matcher(in); - if (matProb.matches()) { - if (!in.contains("*")) { - strFormatR = "PROB_" + matProb.group(1) + matProb.group(2) + matProb.group(3); - } else { - strFormatR = "PROB_" + matProb.group(1); - } - } - - return strFormatR.replace("(", "") - .replace(")", "") - .replace("<=", "le") - .replace(">=", "ge") - .replace("=", "eq") - .replace("<", "lt") - .replace(">", "gt"); - } - - public static String buildTemplateInfoString(final String tmpl, final MVOrderedMap vals, - final MVOrderedMap tmplMaps, - final PrintStream printStream) throws ValidationException { - return buildTemplate(tmpl, vals, tmplMaps, printStream, "infoString"); - } - - public static String buildTemplateString( - final String tmpl, final MVOrderedMap vals, - final MVOrderedMap tmplMaps, - final PrintStream printStream) throws ValidationException { - - return buildTemplate(tmpl, vals, tmplMaps, printStream, "fileName"); - } - - /** - * Populate a template string, specified by tmpl, with values specified in the input map vals. If - * a template tag is not found in the input vals table, a warning is printed and the tag is passed - * through to the output. - * - * @param tmpl Template String containing tags with format - * @param vals Contains a mapping from tag names to values - * @param tmplMaps Map of value maps for each template field, used with map template parm - * (optional) - * @return String built using the template and values - */ - private static String buildTemplate( - final String tmpl, final MVOrderedMap vals, - final MVOrderedMap tmplMaps, - final PrintStream printStream, final String stringType) throws ValidationException { - - - String strRet = tmpl; - try { - Matcher matTmpl = plotTmpl.matcher(tmpl); - SimpleDateFormat formatDate = new SimpleDateFormat("yyyyMMdd", Locale.US); - formatDate.setTimeZone(TimeZone.getTimeZone("UTC")); - while (matTmpl.find()) { - String strTmplTag = matTmpl.group(1); - String strTmplTagName = matTmpl.group(2); - - MVOrderedMap mapParms = parseTagParams(strTmplTag); - if (strTmplTagName.equals("date")) { - vals.put("date", formatDate.format(new Date())); - } - - if (!vals.containsKey(strTmplTagName)) { - if(printStream != null) { - printStream.println(" ** WARNING: template tag " + strTmplTagName + " not found in agg" - + " perm"); - }else { - logger.info(" ** WARNING: template tag " + strTmplTagName + " not found in agg" - + " perm"); - } - continue; - } - - String strVal = (String) vals.get(strTmplTagName); - - - // if there is a corresponding tag value map, use the map value - if (mapParms.containsKey("map")) { - String strMapName = mapParms.get("map").toString(); - if (strMapName.equalsIgnoreCase("true")) { - strMapName = strTmplTagName; - } - MVOrderedMap mapTmplVal = (MVOrderedMap) tmplMaps.get(strMapName); - if (null == mapTmplVal) { - throw new ValidationException( - "template tag " + strTmplTagName + " does not have a val_map defined"); - } - if (mapTmplVal.containsKey(strVal)) { - strVal = mapTmplVal.getStr(strVal); - } + return result; + + } + + + /** + * Pads input str with spaces appended to the end so that the length of the returned String is at + * least width characters + * + * @param str The string to pad + * @param width The minimum number of characters in the returned String + * @return the padded version of the input str + */ + public static String padEnd(String str, final String pad, final int width) { + while (width > str.length()) { + str += pad; + } + return str; + } + + public static String padEnd(final String str, final int width) { + return padEnd(str, " ", width); + } + + + /** + * Pads input str with spaces appended to the beginning so that the length of the returned String + * is at least width characters + * + * @param str The string to pad + * @param width The minimum number of characters in the returned String + * @return the padded version of the input str + */ + public static String padBegin(String str, final String pad, final int width) { + while (width > str.length()) { + str = pad + str; + } + return str; + } + + public static String padBegin(final String str, final int width) { + return padBegin(str, " ", width); + } + + public static String padBegin(final String str) { + return padBegin(str, 16); + } + + /** + * Create a string representation for the input time span, which should represent milliseconds + * between events. For example, a time span message can be generated as follows: + * formatTimeStamp(dateEnd.getTime() - dateStart.getTime()) + * + * @param span Time span, in milliseconds + * @return Time span in format [days]d H:mm:ss.mmmm + */ + public static String formatTimeSpan(long span) { + long intDay = span / (24L * 60L * 60L * 1000L); + span -= intDay * 24L * 60L * 60L * 1000L; + long intHr = span / (60L * 60L * 1000L); + span -= intHr * 60L * 60L * 1000L; + long intMin = span / (60L * 1000L); + span -= intMin * 60L * 1000L; + long intSec = span / 1000L; + span -= intSec * 1000L; + long intMs = span; + + return (0 < intDay ? Long.toString(intDay) + "d " : "") + Long.toString(intHr) + + (10 > intMin ? ":0" : ":") + Long.toString(intMin) + + (10 > intSec ? ":0" : ":") + Long.toString(intSec) + "." + + (100 > intMs ? "0" + (10 > intMs ? "0" : "") : "") + Long.toString(intMs); + } + + /** + * Append the first array with the values of the second + * + * @param l1 Array to be appended + * @param l2 Array to append + * @return The combined array + */ + public static Object[] append(final Object[] l1, final Object[] l2, final Object[] cast) { + List listRet = Arrays.asList(l1); + listRet.addAll(Arrays.asList(l2)); + return listRet.toArray(cast); + + } + + public static String[] append(final String[] s1, final String[] s2) { + return (String[]) append(s1, s2, new String[]{}); + } + + public static String[] append(final String s1, final String[] s2) { + return append(new String[]{s1}, s2); + } + + public static String[] append(final String[] s1, final String s2) { + return append(s1, new String[]{s2}); + } + + + public static Map.Entry[] append(final Map.Entry[] s1, final Map.Entry[] s2) { + return (Map.Entry[]) append(s1, s2, new Map.Entry[]{}); + } + + public static MVPlotJob[] append(final MVPlotJob[] s1, final MVPlotJob[] s2) { + return (MVPlotJob[]) append(s1, s2, new MVPlotJob[]{}); + } + + public static int[] append(int[] s1, int[] s2) { + if (null == s1) { + s1 = new int[]{}; + } + if (null == s2) { + s2 = new int[]{}; + } + int r = 0; + int[] ret = new int[s1.length + s2.length]; + for (int i = 0; i < s1.length; i++) { + ret[r++] = s1[i]; + } + for (int i = 0; i < s2.length; i++) { + ret[r++] = s2[i]; + } + return ret; + } + + public static String[] unique(final String[] dataArr) { + Map table = new HashMap<>(); + for (String data : dataArr) { + if (!table.containsKey(data)) { + table.put(data, "true"); + } + } + Set strings = table.keySet(); + return strings.toArray(new String[strings.size()]); + } + + public static int sum(int[] dataArr) { + if (1 > dataArr.length) { + return 0; + } + int intSum = 0; + for (int data : dataArr) { + intSum += data; + } + return intSum; + } + + /** + * Attempt to convert the input ArrayList, which is assumed to contain all Strings, to a + * String[]. + * + * @param list ArrayList to convert + * @return Converted list + */ + public static String[] toArray(final List list) { + return list.toArray(new String[list.size()]); + } + + public static List toArrayList(final String[] list) { + List ret = new ArrayList<>(); + ret.addAll(Arrays.asList(list)); + return ret; + } + + /** + * Create a deep copy of the input list + * + * @param list List to copy + * @return Copied list + */ + public static MVOrderedMap[] copyList(final MVOrderedMap[] list) { + MVOrderedMap[] listRet = new MVOrderedMap[list.length]; + for (int i = 0; i < list.length; i++) { + listRet[i] = new MVOrderedMap(list[i]); + } + return listRet; + } + + /** + * Create a deep copy of the input list + * + * @param list List to copy + * @return Copied list + */ + public static String[] copyList(final String[] list) { + return list.clone(); + } + + + /** + * Returns a string representation of the MVOrderedMap in R declaration syntax + * + * @param map Data structure to convert to R list representation + * @return The R-syntax representation of the input map + */ + public static String getRDecl(final MVOrderedMap map) { + String strRDecl = "list(\n"; + List keys = map.getListKeys(); + String[] listKeys = (String[]) keys.toArray(new String[keys.size()]); + for (int i = 0; i < listKeys.length; i++) { + if (0 < i) { + strRDecl += ",\n" + MVUtil.padBegin("`" + listKeys[i].replace("&", "&").replace(">", ">") + .replace("<", "<") + "`") + " = "; + } else { + strRDecl += "" + MVUtil.padBegin("`" + listKeys[i].replace("&", "&").replace(">", ">") + .replace("<", "<") + "`") + " = "; + } + Object objVal = map.get(listKeys[i]); + if (objVal instanceof String) { + strRDecl += "\"" + objVal.toString() + "\""; + } else if (objVal instanceof String[]) { + strRDecl += "c("; + String[] listVal = (String[]) objVal; + for (int j = 0; j < listVal.length; j++) { + strRDecl += (0 < j ? ", " : "") + "\"" + listVal[j] + "\""; + } + strRDecl += ")"; + } else if (objVal instanceof MVOrderedMap) { + strRDecl += ((MVOrderedMap) objVal).getRDecl(); + } else { + strRDecl += "\"???\",\n"; + } + } + strRDecl += "\n)"; + return strRDecl; + } + + public static Map getYamlDecl(final MVOrderedMap map) { + Map result = new LinkedHashMap<>(); + List keys = map.getListKeys(); + for (String key : keys) { + Object objVal = map.get(key); + if (objVal instanceof String) { + result.put(key, new String[]{((String) objVal).replace("&", "&").replace(">", ">") + .replace("<", "<")}); + } else if (objVal instanceof String[]) { + result.put(key, objVal); + } else if (objVal instanceof MVOrderedMap) { + result.put(key, ((MVOrderedMap) objVal).getYamlDecl()); + } + } + return result; + } + + + public static String[] parseModeStat(final String stat) { + Matcher mat = _patModeStat.matcher(stat); + if (!mat.matches()) { + return new String[]{stat}; + } + return new String[]{mat.group(1), mat.group(2)}; + } + + + /** + * Determine the database line_data table in which the input statistic is stored. For mode stats, + * consider only the first portion of the stat name. If the stat is not found in any table, + * return an empty string. + * + * @param strStat stat name to look up + * @return the name of the database line_data table which contains the stat + */ + public static String getStatTable(final String strStat) { + String strStatMode = parseModeStat(strStat)[0]; + String strStatMrd = ""; + if (strStatMode.equals(strStat)) { + String[] listStatComp = strStat.split("_"); + strStatMrd = strStat.replace("_" + listStatComp[listStatComp.length - 1], ""); + } + + //TODO mtdRatioField and modeRatioField contain the same fields + + if (statsCnt.containsKey(strStat)) { + return "line_data_cnt"; + } else if (statsCts.containsKey(strStat)) { + return "line_data_cts"; + } else if (statsNbrcnt.containsKey(strStat)) { + return "line_data_nbrcnt"; + } else if (statsNbrcts.containsKey(strStat)) { + return "line_data_nbrcts"; + } else if (statsPstd.containsKey(strStat)) { + return "line_data_pstd"; + } else if (statsMcts.containsKey(strStat)) { + return "line_data_mcts"; + } else if (statsRhist.containsKey(strStat)) { + return "line_data_rhist"; + } else if (statsVl1l2.containsKey(strStat)) { + return "line_data_vl1l2"; + } else if (statsVal1l2.containsKey(strStat)) { + return "line_data_val1l2"; + } else if (modeSingleStatField.containsKey(strStatMode)) { + return "mode_obj_single"; + } else if (modePairStatField.containsKey(strStatMode)) { + return "mode_obj_pair"; + } else if (modeRatioField.contains(strStat)) { + return "mode_obj_single"; + } else if (statsEnscnt.containsKey(strStat)) { + return "line_data_enscnt"; + } else if (statsMpr.containsKey(strStat)) { + return "line_data_mpr"; + } else if (statsOrank.containsKey(strStat)) { + return "line_data_orank"; + } else if (statsSsvar.containsKey(strStat)) { + return "line_data_ssvar"; + } else if (mtd3dSingleStatField.containsKey(strStatMrd)) { + return "mtd_3d_obj_single"; + } else if (mtd3dPairStatField.containsKey(strStatMrd)) { + return "mtd_3d_obj_pair"; + } else if (mtd2dStatField.containsKey(strStatMrd)) { + return "mtd_2d_obj"; + } else if (mtdRatioField.contains(strStat)) { + return "mtd_3d_obj_single"; + } else if (statsVcnt.containsKey(strStat)) { + return "line_data_vcnt"; + } else if (statsEcnt.containsKey(strStat)) { + return "line_data_ecnt"; + } else if (statsPerc.containsKey(strStat)) { + return "line_data_perc"; + } else if (statsDmap.containsKey(strStat)) { + return "line_data_dmap"; + } else if (statsRps.containsKey(strStat)) { + return "line_data_rps"; + } else if (statsCtc.containsKey(strStat)) { + return "line_data_ctc"; + } else if (statsNbrctc.containsKey(strStat)) { + return "line_data_ctc"; + } else if (statsPct.containsKey(strStat)) { + return "line_data_pct"; + } else if (statsSl1l2.containsKey(strStat)) { + return "line_data_sl1l2"; + } else if (statsSal1l2.containsKey(strStat)) { + return "line_data_sal1l2"; + } else if (statsGrad.containsKey(strStat)) { + return "line_data_grad"; + } else if (statsSsidx.containsKey(strStat)) { + return "line_data_ssidx"; + } else if (statsSeeps.containsKey(strStat)) { + return "line_data_seeps"; + } else { + return ""; + } + } + + + public static MvResponse runRscript( + final String rscript, + final String script) { + return runRscript(rscript, script, new String[]{}, null); + } + + public static MvResponse runRscript( + final String rscriptCommand, final String scriptName, + final String[] args) { + return runRscript(rscriptCommand, scriptName, args, null); + } + + /** + * Executes an R script or Python script using the specified command, script name, arguments, + * and environment variables. Captures the standard output and error streams, logs the process, + * and returns the result in an `MvResponse` object. + * + * @param rscriptCommand The command to execute the script (e.g., "Rscript" or "python"). + * @param scriptName The name of the script to execute. + * @param args An array of arguments to pass to the script. + * @param env A map of environment variables to set for the process. + * @return An `MvResponse` object containing the success status, output, and error messages. + */ + public static MvResponse runRscript( + final String rscriptCommand, final String scriptName, + final String[] args, final Map env) { + + MvResponse mvResponse = new MvResponse(); + + // Build a list of arguments from the provided array + StringBuilder argList = new StringBuilder(); + for (int i = 0; null != args && i < args.length; i++) { + argList.append(' ').append(args[i]); } - if (stringType.equals("fileName")) { - strVal = strVal.replace(">", "gt").replace("<", "lt").replaceAll("=", "e"); + + Process proc = null; + InputStreamReader inputStreamReader = null; + InputStreamReader errorInputStreamReader = null; + + BoundedBufferedReader readerProcStd = null; + BoundedBufferedReader readerProcErr = null; + + boolean boolExit = false; + int intExitStatus = 0; + StringBuilder strProcStd = new StringBuilder(); + StringBuilder strProcErr = new StringBuilder(); + InputStream error = null; + InputStream in = null; + + // Clean and prepare the command, script name, and arguments + String rscriptCommandClean = cleanString(rscriptCommand); + String scriptNameClean = cleanString(scriptName); + String strArgListClean = cleanString(argList.toString()); + List command = new ArrayList<>(); + command.add(rscriptCommandClean.trim()); + command.add(scriptNameClean.trim()); + command.add(strArgListClean.trim()); + logger.info("Running command: " + String.join(" ", command)); + + // Create a ProcessBuilder and set the environment variables + ProcessBuilder pb = new ProcessBuilder(command); + if(env != null) { + pb.environment().putAll(env); } - // if there is a format parameter, apply it to the value - if (mapParms.containsKey("format")) { - String strFormat = mapParms.getStr("format"); - if (strTmplTagName.equals("fcst_lead")) { - if (strVal.equals("0")) { - strVal = "00000"; + try { + // Start the process and wait for it to complete + proc = pb.start(); + int errorCode = proc.waitFor(); + + // Capture the error and standard output streams + error = proc.getErrorStream(); + errorInputStreamReader = new InputStreamReader(error); + in = proc.getInputStream(); + inputStreamReader = new InputStreamReader(in); + + readerProcStd = new BoundedBufferedReader(inputStreamReader, 50000, 2048); + readerProcErr = new BoundedBufferedReader(errorInputStreamReader, 50000, 2048); + + // Read the standard output + while (readerProcStd.ready()) { + String line = readerProcStd.readLineBounded(); + strProcStd.append(line).append('\n'); } - if (strFormat.equals("HH")) { - strVal = strVal.substring(0, strVal.length() - 4); + + // Read the error output + while (readerProcErr.ready()) { + String line = readerProcErr.readLineBounded(); + strProcErr.append(line).append('\n'); } - if (strFormat.equals("HHmm")) { - strVal = strVal.substring(0, strVal.length() - 2); + + // Destroy the process after execution + proc.destroy(); + + } catch (SecurityException | IOException | IllegalArgumentException | InterruptedException e) { + // Log any exceptions that occur during process execution + logger.error(e.getMessage()); + } finally { + // Close all streams and clean up resources + if (in != null) { + try { + in.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } } - while (strFormat.length() > strVal.length()) { - strVal = "0" + strVal; + if (error != null) { + try { + error.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } } - } else if (strTmplTagName.equals("init_hour") - || strTmplTagName.equals("valid_hour") && strFormat.equals("HH")) { - while (2 > strVal.length()) { - strVal = "0" + strVal; + if (inputStreamReader != null) { + try { + inputStreamReader.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } } - } + if (errorInputStreamReader != null) { + try { + errorInputStreamReader.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + if (readerProcStd != null) { + try { + readerProcStd.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + if (readerProcErr != null) { + try { + readerProcErr.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + if (proc != null) { + proc.destroy(); + } + + } + + // Determine the script type (Rscript or Python) based on the command + String type = "Rscript"; + if (rscriptCommand.contains("python")) { + type = PYTHON; + } + // Append the standard output to the response + if (!strProcStd.isEmpty()) { + mvResponse.setInfoMessage( + "\n==== Start " + type + " output ====\n" + strProcStd + "==== End " + type + " output ====\n"); + } + + // Append the error output to the response, if any + if (!strProcErr.isEmpty()) { + String[] errorLines = strProcErr.toString().split("\\r?\\n|\\r"); + mvResponse.setInfoMessage( + mvResponse.getInfoMessage() + "\n==== Start " + type + " ERROR ====\n" + errorLines[errorLines.length - 1] + "\n==== End " + type + " error ====\n" + + ); + + mvResponse.setErrorMessage( + "==== Start " + type + " error ====\n" + strProcErr + "==== End " + type + " ERROR ===="); + } + // Set the success status of the response + mvResponse.setSuccess(0 == intExitStatus); + return mvResponse; + } + + /** + * Populate the template tags in the input template file named tmpl with values from the input + * table vals and write the result to the output file named output. + * + * @param tmpl Template file to populate + * @param output Output file to write + * @param vals Table containing values corresponding to tags in the input template + * @throws Exception + */ + public static void populateTemplateFile( + final String tmpl, final String output, + final Map vals) throws IOException { + try (FileReader fileReader = new FileReader(tmpl); + BoundedBufferedReader reader = new BoundedBufferedReader(fileReader); + PrintStream writer = new PrintStream(output)) { + while (reader.ready()) { + String strTmplLine = reader.readLineBounded(); + String strOutputLine = strTmplLine; + + Matcher matRtmplLine = patRTmpl.matcher(strTmplLine); + while (matRtmplLine.find()) { + String strRtmplTag = matRtmplLine.group(1); + if (!vals.containsKey(strRtmplTag)) { + continue; + } + String strRTagVal = (String) vals.get(strRtmplTag); + if (strRTagVal != null) { + strOutputLine = strOutputLine.replace("#<" + strRtmplTag + ">#", strRTagVal); + } + + } + + writer.println(strOutputLine); + } + } catch (IOException e) { + logger.error(e.getMessage()); + throw e; + } + + } + + + /** + * Build a list of the fcst_var/stat combinations stored in the input structure. The output + * list is structured as a list of pairs of Strings, with the first element storing the fcst_var + * and the second element storing the associated statistic. + * + * @param mapDep or structure from a MVPlotJob + * @return a list of fcst_var/stat pairs + */ + public static String[][] buildFcstVarStatList(final MVOrderedMap mapDep) { + List listRet = new ArrayList(); + String[] listFcstVar = mapDep.getKeyList(); + String[] listStat; + for (int intFcstVar = 0; intFcstVar < listFcstVar.length; intFcstVar++) { + listStat = (String[]) mapDep.get(listFcstVar[intFcstVar]); + for (int intStat = 0; intStat < listStat.length; intStat++) { + listRet.add(new String[]{listFcstVar[intFcstVar], listStat[intStat]}); + } + } + return (String[][]) listRet.toArray(new String[][]{}); + } + + + /** + * Construct the template map for the specified permutation of plot_fix values, using the + * specified set values. + * + * @param mapPlotFix plot_fix field/value pairs to use in populating the template values + * @param mapPlotFixVal values used for sets + * @throws Exception + */ + public static Map.Entry[] buildPlotFixTmplMap( + final MVOrderedMap mapPlotFix, + final MVOrderedMap mapPlotFixVal) { + Map.Entry[] listPlotFixVal = mapPlotFix.getOrderedEntries(); + // replace fixed value set names with their value maps + ArrayList listPlotFixValAdj = new ArrayList(); + for (Map.Entry aListPlotFixVal : listPlotFixVal) { + String strFixVar = aListPlotFixVal.getKey().toString(); + if (!strFixVar.endsWith("_set")) { + listPlotFixValAdj.add(aListPlotFixVal); + continue; + } + + String strFixVarAdj = strFixVar.replaceAll("_set$", ""); + MVOrderedMap mapFixSet = (MVOrderedMap) mapPlotFixVal.get(strFixVarAdj); + listPlotFixValAdj.add(new MVMapEntry(strFixVarAdj, mapFixSet)); + } + listPlotFixVal = (Map.Entry[]) listPlotFixValAdj + .toArray(new Map.Entry[listPlotFixValAdj.size()]); + + return listPlotFixVal; + } + + /** + * check if the aggregation type is compatible with the statistic We're currently only aggregating + * SL1L2 -> CNT and CTC -> CTS + * + * @param tableStats + * @param strStat + * @param aggType + * @throws Exception + */ + public static void isAggTypeValid( + final Map tableStats, final String strStat, + final String aggType) throws ValidationException { + //check if aggType is allowed for this stat + String[] types = tableStats.get(strStat); + boolean isFound = false; + for (String type : types) { + if (type.equals(aggType)) { + isFound = true; + break; + } + } + if (!isFound) { + throw new ValidationException( + "aggregation type " + aggType + " isn't compatible with the statistic " + strStat); + } + } - if (mapParms.getStr("format").equalsIgnoreCase("R")) { - strVal = MVUtil.replaceSpecialChars(strVal); - } + public static String findValue( + final String[] listToken, final List headerNames, + final String header) { + int pos = headerNames.indexOf(header); + if (pos >= 0 && pos < listToken.length) { + return listToken[pos]; + } else { + return "NA"; } + } + - // if the tag value is a date, format it accordingly + public static void updateLog4jConfiguration() { try { - SimpleDateFormat formatDb = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US); - formatDb.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat formatDBms = new SimpleDateFormat(MVUtil.DB_DATE_MS, Locale.US); - formatDBms.setTimeZone(TimeZone.getTimeZone("UTC")); - Date dateParse = formatDb.parse(strVal); - if (null != dateParse) { - strVal = formatPlotFormat(dateParse); - } else { - dateParse = formatDBms.parse(strVal); - if (null != dateParse) { - strVal = formatPlotFormat(dateParse); + String jarPath = MVUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI() + .getPath(); + URI imgurl = new URI("jar:file:" + jarPath + "!/edu/ucar/metviewer/resources/log4j2.xml"); + Logger l = (Logger) LogManager.getLogger(LogManager.ROOT_LOGGER_NAME); + l.getContext().setConfigLocation(imgurl); + } catch (URISyntaxException | SecurityException | UnsupportedOperationException e) { + logger.error(e.getMessage()); + } + + } + + /** + * Format the input String so that it conforms to R variable name standards + * + * @param in String to format + * @return Formatted String + */ + public static String replaceSpecialChars(final String in) { + + String strFormatR = in; + Matcher matProb = MVUtil.prob.matcher(in); + if (matProb.matches()) { + if (!in.contains("*")) { + strFormatR = "PROB_" + matProb.group(1) + matProb.group(2) + matProb.group(3); + } else { + strFormatR = "PROB_" + matProb.group(1); } - } - } catch (ParseException e) { - logger.debug(e.getMessage()); - } - - // if the tag is a threshold, format it accordingly - if (strTmplTagName.equals("fcst_thresh") || strTmplTagName.equals("fcst_thr") - || strTmplTagName.equals("obs_thresh") || strTmplTagName.equals("obs_thr")) { - strVal = formatThresh(strTmplTag, strVal); - } - //replace "/" with "_" - file names can't have "/"!!!!! - strVal = strVal.replace("/", "_"); - - strRet = strRet.replace("{" + strTmplTag + "}", strVal); - } - }catch (Exception e){ - System.out.println(e.getMessage()); - } - return strRet; - } - - /** - * Parse template tag parameter pairs and return them in an ordered map. For example, - * parseTagParams("tag_name?param1=val1;param2=val2") returns a map with two members, - * param1 and param2 with their values set accordingly. - * - * @param tag Formatted tag with param/value pairs to parse - * @return Ordered map containing parsed param/value pairs - */ - public static MVOrderedMap parseTagParams(final String tag) { - MVOrderedMap mapRet = new MVOrderedMap(); - Matcher mat = MVUtil.tag.matcher(tag); - if (mat.matches() && null != mat.group(2)) { - String[] listPairs = mat.group(2).split("\\s*&\\s*"); - for (int i = 0; i < listPairs.length; i++) { - String[] listPair = listPairs[i].split("\\s*=\\s*"); - mapRet.put(listPair[0], listPair[1]); - } - } - - return mapRet; - } - - /** - * Reformat the fcst_thresh value using the directions provided in the body of the template tag. - * It is assumed that the input template tag has the parameterized tag format: - * fcst_thresh?param1=val1;param2=val2[;...] where the params can be the following:
    - *
  • units set to either mm or in (input assumed to be in mm)
  • format set to - * the java formatting string to apply, for example 0.00#
  • symbol set to either letters - * or math, for example ge or >=, respectively
- * - * @param fcstTag Template tag name (including params) for fcst_thresh - * @param fcstThresh Template map value to be formatted - * @return - */ - public static String formatThresh(final String fcstTag, final String fcstThresh) { - String strThreshRet = fcstThresh; - MVOrderedMap mapParams = parseTagParams(fcstTag); - DecimalFormat format = new DecimalFormat("0.000"); - - // attempt to parse the input threshold - String strSymbol; - String strThresh; - double dblThresh; - Matcher matFcstThresh = MVUtil.thresh.matcher(fcstThresh); - if (matFcstThresh.matches()) { - strSymbol = matFcstThresh.group(1); - strThresh = matFcstThresh.group(2); - dblThresh = Double.parseDouble(strThresh); - } else { - return strThreshRet; - } - - // change the units, if requested - if (mapParams.containsKey("units")) { - String strUnits = mapParams.get("units").toString(); - if (strUnits.equals("in")) { - strThresh = format.format(dblThresh /= 25.4); - strThreshRet = strSymbol + strThresh; - } - } - - // change the format, if requested - if (mapParams.containsKey("format")) { - String strFormat = mapParams.get("format").toString(); - strThresh = new DecimalFormat(strFormat).format(dblThresh); - strThreshRet = strSymbol + strThresh; - } - - // change the logic symbol, if requested - if (mapParams.containsKey("symbol")) { - String strSymbolType = mapParams.get("symbol").toString(); - if (strSymbolType.equals("letters")) { - strSymbol = strSymbol.replace("==", "eq") - .replace("!=", "ne") + } + + return strFormatR.replace("(", "") + .replace(")", "") .replace("<=", "le") .replace(">=", "ge") + .replace("=", "eq") .replace("<", "lt") .replace(">", "gt"); - strThreshRet = strSymbol + strThresh; - } - } - - return strThreshRet; - } - - public static String buildTemplateString( - final String tmpl, final MVOrderedMap vals, - final PrintStream printStream) throws ValidationException { - return buildTemplateString(tmpl, vals, null, printStream); - } - - /** - * Add the fcst_var and stat names from the dep structure of the input job to the input list of - * tmpl map values. - * - * @param job MVPlotJob whose dep structure will be processed - */ - public static MVOrderedMap addTmplValDep(MVPlotJob job) { - MVOrderedMap mapTmplValsPlot = new MVOrderedMap(job.getTmplVal()); - if (job.getIndyVar() != null) { - mapTmplValsPlot.put("indy_var", job.getIndyVar()); - } - for (int intY = 1; intY <= 2; intY++) { - - // get a list of dep groups - MVOrderedMap[] listDepGroup = job.getDepGroups(); - MVOrderedMap mapDep = null; - if (listDepGroup.length > 0) { - mapDep = (MVOrderedMap) listDepGroup[0].get("dep" + intY); - } - if (mapDep != null) { - Map.Entry[] listDep = mapDep.getOrderedEntries(); - - // build tmpl map values for each fcst_var - String strDep = "dep" + intY; - for (int i = 0; i < listDep.length; i++) { - - // resolve the fcst_var and stats for the current dep - String strFcstVar = listDep[i].getKey().toString(); - String[] listStat = (String[]) listDep[i].getValue(); - - // build and add the fcst_var to the tmpl value map - String strDepFcstVar = strDep + "_" + (i + 1); - mapTmplValsPlot.put(strDepFcstVar, strFcstVar); - for (int j = 0; j < listStat.length; j++) { - mapTmplValsPlot.put(strDepFcstVar + "_stat" + (j + 1), listStat[j]); - } - } - } - } - return mapTmplValsPlot; - } - - /** - * Creates a string representation of an R collection containing the list of values in the input - * list, val. - * - * @param val List of values to print in the R collection - * @param ticks (optional) Print tickmarks around values, for when constituents are factors as - * opposed to numeric, defaults to true - * @return String representation of the R collection - */ - public static String printRCol(final Object[] val, final boolean ticks) { - StringBuilder rStr = new StringBuilder("c("); - for (int i = 0; i < val.length; i++) { - if (0 < i) { - rStr.append(", "); - } - String value = val[i].toString().replace("&", "&").replace(">", ">") - .replace("<", "<"); - if (ticks) { - rStr.append("\"").append(value).append("\""); - } else { - rStr.append(value); - } - } - rStr.append(")"); - return rStr.toString(); - } - - /** - * Creates a string representation of an R collection containing the list of values in the input - * list, val. - * - * @param val List of values to print in the R collection - * @return String representation of the R collection - */ - public static String[] printYamlCol(final Object[] val) { - List result = new ArrayList<>(); - for (Object o : val) { - String value = o.toString().replace("&", "&").replace(">", ">") - .replace("<", "<"); - result.add(value); - } - return result.toArray(new String[0]); - } - - public static String printRCol(final Object[] val) { - return printRCol(val, true); - } - - /** - * Parses an R collection string representation and returns a list of the values in the - * collection. An R collection has syntax c(1, 2, 3) or c("a", "bb", "ccc") - * - * @param strRCol - * @return list of String representations of each collection member - */ - public static String[] parseRCol(final String strRCol) { - if (strRCol.contains("\"")) { - Matcher matRColStr = Pattern.compile("c\\(\\s*\"(.*)\"\\s*\\)").matcher(strRCol); - if (!matRColStr.matches()) { - return new String[]{}; - } - String strList = matRColStr.group(1); - List list = new ArrayList<>(); - while (strList.matches(".*\"\\s*,\\s*\".*")) { - list.add(strList.replaceFirst("\"\\s*,\\s*\".*", "")); - strList = strList.replaceFirst(".*?\"\\s*,\\s*\"", ""); - } - list.add(strList.replaceFirst("\"\\s*,\\s*\".*", "")); - return list.toArray(new String[list.size()]); - } else { - Matcher matRColNum = Pattern.compile("c\\(\\s*(.*)\\s*\\)").matcher(strRCol); - if (!matRColNum.matches()) { - return new String[]{}; - } - String strList = matRColNum.group(1); - return strList.split("\\s*,\\s*"); - } - } - - private static void populateInt(Map tableRTags, String tagName, String strValue, int def) { - int valInt = def; - try { - valInt = Integer.parseInt(strValue); - } catch (Exception e) { - } - tableRTags.put(tagName, valInt); - } - - private static void populateDouble(Map tableRTags, String tagName, String strValue, double def) { - double valDouble = def; - try { - valDouble = Double.parseDouble(strValue); - } catch (Exception e) { - } - tableRTags.put(tagName, valDouble); - } - - - /** - * Populate the input table with the plot formatting tag values stored in the input job. - * - * @param tableRTags template value table to receive plot formatting values - * @param job source for plot formatting values - */ - public static void populatePlotFmtTmplYaml(Map tableRTags, MVPlotJob job) { - populateDouble(tableRTags, "alpha", job.getCIAlpha(), 0.05); - - tableRTags.put("plot_type", job.getPlotType()); - populateDouble(tableRTags, "plot_width", job.getPlotWidth(), 11); - populateDouble(tableRTags, "plot_height", job.getPlotHeight(), 8.5); - populateInt(tableRTags, "plot_res", job.getPlotRes(), 72); - - tableRTags.put("plot_units", job.getPlotUnits()); - - String[] valArr = job.getMar().replace("c(", "").replace(")", "").split(","); - List vals = new ArrayList<>(); - try { - for (String s : valArr) { - vals.add(Double.parseDouble(s)); - } - } catch (Exception e) { - } - tableRTags.put("mar", vals); - - valArr = job.getMgp().replace("c(", "").replace(")", "").split(","); - vals = new ArrayList<>(); - try { - for (String s : valArr) { - vals.add(Double.parseDouble(s)); - } - } catch (Exception e) { - } - tableRTags.put("mgp", vals); - - populateInt(tableRTags, "cex", job.getCex(), 1); - populateDouble(tableRTags, "title_weight", job.getTitleWeight(), 1.4); - populateDouble(tableRTags, "title_size", job.getTitleSize(), 1.4); - populateInt(tableRTags, "title_offset", job.getTitleOffset(), -2); - populateDouble(tableRTags, "title_align", job.getTitleAlign(), 0.5); - - populateInt(tableRTags, "xtlab_orient", job.getXtlabOrient(), 1); - populateDouble(tableRTags, "xtlab_perp", job.getXtlabPerp(), 0.75); - populateDouble(tableRTags, "xtlab_horiz", job.getXtlabHoriz(), 0.5); - populateInt(tableRTags, "xtlab_decim", job.getXtlabFreq(), 1); - populateInt(tableRTags, "xtlab_size", job.getXtlabSize(), 1); - - populateInt(tableRTags, "xlab_weight", job.getXlabWeight(), 1); - populateInt(tableRTags, "xlab_size", job.getXlabSize(), 1); - populateInt(tableRTags, "xlab_offset", job.getXlabOffset(), 2); - populateDouble(tableRTags, "xtlab_horiz", job.getXtlabHoriz(), 0.5); - populateDouble(tableRTags, "xlab_align", job.getXlabAlign(), 0.5); - - populateInt(tableRTags, "ytlab_orient", job.getYtlabOrient(), 1); - populateDouble(tableRTags, "ytlab_perp", job.getYtlabPerp(), 0.5); - populateDouble(tableRTags, "ytlab_horiz", job.getYtlabHoriz(), 0.5); - populateInt(tableRTags, "ytlab_size", job.getYtlabSize(), 1); - populateInt(tableRTags, "ylab_weight", job.getYlabWeight(), 1); - populateInt(tableRTags, "ylab_size", job.getYlabSize(), 1); - populateInt(tableRTags, "ylab_offset", job.getYlabOffset(), -2); - populateDouble(tableRTags, "ylab_align", job.getYlabAlign(), 0.5); - - populateInt(tableRTags, "grid_lty", job.getGridLty(), 3); - tableRTags.put("grid_col", job.getGridCol()); - populateInt(tableRTags, "grid_lwd", job.getGridLwd(), 1); - tableRTags.put("grid_x", job.getGridX()); - - populateInt(tableRTags, "x2tlab_orient", job.getX2tlabOrient(), 1); - populateInt(tableRTags, "x2tlab_perp", job.getX2tlabPerp(), 1); - populateDouble(tableRTags, "x2tlab_horiz", job.getX2tlabHoriz(), 0.5); - populateDouble(tableRTags, "x2tlab_size", job.getX2tlabSize(), 0.8); - populateInt(tableRTags, "x2lab_weight", job.getX2labWeight(), 1); - populateDouble(tableRTags, "x2lab_size", job.getX2labSize(), 0.8); - populateDouble(tableRTags, "x2lab_offset", job.getX2labOffset(), -0.5); - populateDouble(tableRTags, "x2lab_align", job.getX2labAlign(), 0.5); - populateInt(tableRTags, "y2tlab_orient", job.getY2tlabOrient(), 1); - populateInt(tableRTags, "y2tlab_perp", job.getY2tlabPerp(), 1); - populateDouble(tableRTags, "y2tlab_horiz", job.getY2tlabHoriz(), 0.5); - populateDouble(tableRTags, "y2tlab_size", job.getY2tlabSize(), 0.5); - populateInt(tableRTags, "y2lab_weight", job.getY2labWeight(), 1); - populateInt(tableRTags, "y2lab_size", job.getY2labSize(), 1); - populateInt(tableRTags, "y2lab_offset", job.getY2labOffset(), 1); - populateDouble(tableRTags, "y2lab_align", job.getY2labAlign(), 0.5); - - populateDouble(tableRTags, "legend_size", job.getLegendSize(), 0.8); - tableRTags.put("legend_box", job.getLegendBox()); - String[] insetStr = job.getLegendInset().replace("c(", "") - .replace(")", "").split(","); - Map insetMap = new HashMap<>(); - try { - insetMap.put("x", Double.valueOf(insetStr[0])); - insetMap.put("y", Double.valueOf(insetStr[1])); - } catch (Exception e) { - insetMap = new HashMap<>(); - } - if (!insetMap.isEmpty()) { - tableRTags.put("legend_inset", insetMap); - } - populateInt(tableRTags, "legend_ncol", job.getLegendNcol(), 3); - populateInt(tableRTags, "caption_weight", job.getCaptionWeight(), 1); - - tableRTags.put("caption_col", job.getCaptionCol()); - populateDouble(tableRTags, "caption_size", job.getCaptionSize(), 0.8); - populateDouble(tableRTags, "caption_offset", job.getCaptionOffset(), 3); - populateDouble(tableRTags, "caption_align", job.getCaptionAlign(), 0); - - tableRTags.put("box_pts", job.getBoxPts().equalsIgnoreCase("true") ? "True" : "False"); - tableRTags.put("box_outline", job.getBoxOutline().equalsIgnoreCase("true") ? "True" : "False"); - populateDouble(tableRTags, "box_boxwex", job.getBoxBoxwex(), 0.2); - tableRTags.put("box_notch", job.getBoxNotch().equalsIgnoreCase("true") ? "True" : "False"); - tableRTags.put("box_avg", job.getBoxAvg().equalsIgnoreCase("true") ? "True" : "False"); - - } - - - - - - /** - * Populate the input table with the plot formatting tag values stored in the input job. - * - * @param tableRTags template value table to receive plot formatting values - * @param job source for plot formatting values - */ - public static void populatePlotFmtTmpl(Map tableRTags, MVPlotJob job) { - tableRTags.put("plot_type", job.getPlotType()); - tableRTags.put("plot_width", job.getPlotWidth()); - tableRTags.put("plot_height", job.getPlotHeight()); - tableRTags.put("plot_res", job.getPlotRes()); - tableRTags.put("plot_units", job.getPlotUnits()); - tableRTags.put("mar", job.getMar()); - tableRTags.put("mgp", job.getMgp()); - tableRTags.put("cex", job.getCex()); - tableRTags.put("title_weight", job.getTitleWeight()); - tableRTags.put("title_size", job.getTitleSize()); - tableRTags.put("title_offset", job.getTitleOffset()); - tableRTags.put("title_align", job.getTitleAlign()); - tableRTags.put("xtlab_orient", job.getXtlabOrient()); - tableRTags.put("xtlab_perp", job.getXtlabPerp()); - tableRTags.put("xtlab_horiz", job.getXtlabHoriz()); - tableRTags.put("xtlab_decim", job.getXtlabFreq()); - tableRTags.put("xtlab_size", job.getXtlabSize()); - tableRTags.put("xlab_weight", job.getXlabWeight()); - tableRTags.put("xlab_size", job.getXlabSize()); - tableRTags.put("xlab_offset", job.getXlabOffset()); - tableRTags.put("xlab_align", job.getXlabAlign()); - tableRTags.put("ytlab_orient", job.getYtlabOrient()); - tableRTags.put("ytlab_perp", job.getYtlabPerp()); - tableRTags.put("ytlab_horiz", job.getYtlabHoriz()); - tableRTags.put("ytlab_size", job.getYtlabSize()); - tableRTags.put("ylab_weight", job.getYlabWeight()); - tableRTags.put("ylab_size", job.getYlabSize()); - tableRTags.put("ylab_offset", job.getYlabOffset()); - tableRTags.put("ylab_align", job.getYlabAlign()); - tableRTags.put("grid_lty", job.getGridLty()); - tableRTags.put("grid_col", job.getGridCol()); - tableRTags.put("grid_lwd", job.getGridLwd()); - tableRTags.put("grid_x", job.getGridX()); - tableRTags.put("x2tlab_orient", job.getX2tlabOrient()); - tableRTags.put("x2tlab_perp", job.getX2tlabPerp()); - tableRTags.put("x2tlab_horiz", job.getX2tlabHoriz()); - tableRTags.put("x2tlab_size", job.getX2tlabSize()); - tableRTags.put("x2lab_weight", job.getX2labWeight()); - tableRTags.put("x2lab_size", job.getX2labSize()); - tableRTags.put("x2lab_offset", job.getX2labOffset()); - tableRTags.put("x2lab_align", job.getX2labAlign()); - tableRTags.put("y2tlab_orient", job.getY2tlabOrient()); - tableRTags.put("y2tlab_perp", job.getY2tlabPerp()); - tableRTags.put("y2tlab_horiz", job.getY2tlabHoriz()); - tableRTags.put("y2tlab_size", job.getY2tlabSize()); - tableRTags.put("y2lab_weight", job.getY2labWeight()); - tableRTags.put("y2lab_size", job.getY2labSize()); - tableRTags.put("y2lab_offset", job.getY2labOffset()); - tableRTags.put("y2lab_align", job.getY2labAlign()); - tableRTags.put("legend_size", job.getLegendSize()); - tableRTags.put("legend_box", job.getLegendBox()); - tableRTags.put("legend_inset", job.getLegendInset()); - tableRTags.put("legend_ncol", job.getLegendNcol()); - tableRTags.put("caption_weight", job.getCaptionWeight()); - tableRTags.put("caption_col", job.getCaptionCol()); - tableRTags.put("caption_size", job.getCaptionSize()); - tableRTags.put("caption_offset", job.getCaptionOffset()); - tableRTags.put("caption_align", job.getCaptionAlign()); - tableRTags.put("box_pts", job.getBoxPts()); - tableRTags.put("box_outline", job.getBoxOutline()); - tableRTags.put("box_boxwex", job.getBoxBoxwex()); - tableRTags.put("box_notch", job.getBoxNotch()); - tableRTags.put("box_avg", job.getBoxAvg()); - tableRTags.put("rely_event_hist", job.getRelyEventHist()); - tableRTags.put("ci_alpha", job.getCIAlpha()); - - } - - /** - * Creates a list of length rep of copies of the input val. Mimics the R function of the same - * name - * - * @param val Value to repeat - * @param rep Number of time to repeat - * @return List of repeated values, with length specified by input rep - */ - public static String[] rep(final String val, final int rep) { - if (1 > rep) { - return new String[]{}; - } - String[] listRet = new String[rep]; - for (int i = 0; i < rep; i++) { - listRet[i] = val; - } - return listRet; - } - - /** - * Creates a list of length rep of copies of the input val. Mimics the R function of the same - * name - * - * @param val Value to repeat - * @param rep Number of time to repeat - * @return List of repeated values, with length specified by input rep - */ - public static Integer[] rep(final int val, final int rep) { - if (1 > rep) { - return new Integer[]{}; - } - Integer[] listRet = new Integer[rep]; - for (int i = 0; i < rep; i++) { - listRet[i] = val; - } - return listRet; - } - - /** - * Creates a list of integers where the first element is min, the second is min+1 .... the last is - * max name - * - * @param min The first value - * @param max The last number - * @return List of values - */ - public static Integer[] repPlusOne(final int min, final int max) { - - Integer[] listRet = new Integer[max - min + 1]; - int start = min; - for (int i = 0; i < (max - min + 1); i++) { - listRet[i] = start; - start++; - } - return listRet; - } - - public static boolean isValidLineType(final String lineType) { - boolean result = false; - for (String type : lineTypes) { - if (type.equalsIgnoreCase(lineType)) { - result = true; - break; - } - } - return result; - } - - public static boolean isNumeric(final Object obj) { - try { - Double.parseDouble(String.valueOf(obj)); - } catch (NumberFormatException nfe) { - return false; - } - return true; - } - - - public static boolean isInteger(String s, int radix) { - if (s.isEmpty()) { - return false; - } - for (int i = 0; i < s.length(); i++) { - if (i == 0 && s.charAt(i) == '-') { - if (s.length() == 1) { - return false; + } + + public static String buildTemplateInfoString(final String tmpl, final MVOrderedMap vals, + final MVOrderedMap tmplMaps, + final PrintStream printStream) throws ValidationException { + return buildTemplate(tmpl, vals, tmplMaps, printStream, "infoString"); + } + + public static String buildTemplateString( + final String tmpl, final MVOrderedMap vals, + final MVOrderedMap tmplMaps, + final PrintStream printStream) throws ValidationException { + + return buildTemplate(tmpl, vals, tmplMaps, printStream, "fileName"); + } + + /** + * Populate a template string, specified by tmpl, with values specified in the input map vals. If + * a template tag is not found in the input vals table, a warning is printed and the tag is passed + * through to the output. + * + * @param tmpl Template String containing tags with format + * @param vals Contains a mapping from tag names to values + * @param tmplMaps Map of value maps for each template field, used with map template parm + * (optional) + * @return String built using the template and values + */ + private static String buildTemplate( + final String tmpl, final MVOrderedMap vals, + final MVOrderedMap tmplMaps, + final PrintStream printStream, final String stringType) throws ValidationException { + + + String strRet = tmpl; + try { + Matcher matTmpl = plotTmpl.matcher(tmpl); + SimpleDateFormat formatDate = new SimpleDateFormat("yyyyMMdd", Locale.US); + formatDate.setTimeZone(TimeZone.getTimeZone("UTC")); + while (matTmpl.find()) { + String strTmplTag = matTmpl.group(1); + String strTmplTagName = matTmpl.group(2); + + MVOrderedMap mapParms = parseTagParams(strTmplTag); + if (strTmplTagName.equals("date")) { + vals.put("date", formatDate.format(new Date())); + } + + if (!vals.containsKey(strTmplTagName)) { + if (printStream != null) { + printStream.println(" ** WARNING: template tag " + strTmplTagName + " not found in agg" + + " perm"); + } else { + logger.info(" ** WARNING: template tag " + strTmplTagName + " not found in agg" + + " perm"); + } + continue; + } + + String strVal = (String) vals.get(strTmplTagName); + + + // if there is a corresponding tag value map, use the map value + if (mapParms.containsKey("map")) { + String strMapName = mapParms.get("map").toString(); + if (strMapName.equalsIgnoreCase("true")) { + strMapName = strTmplTagName; + } + MVOrderedMap mapTmplVal = (MVOrderedMap) tmplMaps.get(strMapName); + if (null == mapTmplVal) { + throw new ValidationException( + "template tag " + strTmplTagName + " does not have a val_map defined"); + } + if (mapTmplVal.containsKey(strVal)) { + strVal = mapTmplVal.getStr(strVal); + } + + } + if (stringType.equals("fileName")) { + strVal = strVal.replace(">", "gt").replace("<", "lt").replaceAll("=", "e"); + } + + // if there is a format parameter, apply it to the value + if (mapParms.containsKey("format")) { + String strFormat = mapParms.getStr("format"); + + if (strTmplTagName.equals("fcst_lead")) { + if (strVal.equals("0")) { + strVal = "00000"; + } + if (strFormat.equals("HH")) { + strVal = strVal.substring(0, strVal.length() - 4); + } + if (strFormat.equals("HHmm")) { + strVal = strVal.substring(0, strVal.length() - 2); + } + while (strFormat.length() > strVal.length()) { + strVal = "0" + strVal; + } + + } else if (strTmplTagName.equals("init_hour") + || strTmplTagName.equals("valid_hour") && strFormat.equals("HH")) { + while (2 > strVal.length()) { + strVal = "0" + strVal; + } + } + + + if (mapParms.getStr("format").equalsIgnoreCase("R")) { + strVal = MVUtil.replaceSpecialChars(strVal); + } + } + + // if the tag value is a date, format it accordingly + + try { + SimpleDateFormat formatDb = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US); + formatDb.setTimeZone(TimeZone.getTimeZone("UTC")); + SimpleDateFormat formatDBms = new SimpleDateFormat(MVUtil.DB_DATE_MS, Locale.US); + formatDBms.setTimeZone(TimeZone.getTimeZone("UTC")); + Date dateParse = formatDb.parse(strVal); + if (null != dateParse) { + strVal = formatPlotFormat(dateParse); + } else { + dateParse = formatDBms.parse(strVal); + if (null != dateParse) { + strVal = formatPlotFormat(dateParse); + } + } + } catch (ParseException e) { + logger.debug(e.getMessage()); + } + + // if the tag is a threshold, format it accordingly + if (strTmplTagName.equals("fcst_thresh") || strTmplTagName.equals("fcst_thr") + || strTmplTagName.equals("obs_thresh") || strTmplTagName.equals("obs_thr")) { + strVal = formatThresh(strTmplTag, strVal); + } + //replace "/" with "_" - file names can't have "/"!!!!! + strVal = strVal.replace("/", "_"); + + strRet = strRet.replace("{" + strTmplTag + "}", strVal); + } + } catch (Exception e) { + System.out.println(e.getMessage()); + } + return strRet; + } + + /** + * Parse template tag parameter pairs and return them in an ordered map. For example, + * parseTagParams("tag_name?param1=val1;param2=val2") returns a map with two members, + * param1 and param2 with their values set accordingly. + * + * @param tag Formatted tag with param/value pairs to parse + * @return Ordered map containing parsed param/value pairs + */ + public static MVOrderedMap parseTagParams(final String tag) { + MVOrderedMap mapRet = new MVOrderedMap(); + Matcher mat = MVUtil.tag.matcher(tag); + if (mat.matches() && null != mat.group(2)) { + String[] listPairs = mat.group(2).split("\\s*&\\s*"); + for (int i = 0; i < listPairs.length; i++) { + String[] listPair = listPairs[i].split("\\s*=\\s*"); + mapRet.put(listPair[0], listPair[1]); + } + } + + return mapRet; + } + + /** + * Reformat the fcst_thresh value using the directions provided in the body of the template tag. + * It is assumed that the input template tag has the parameterized tag format: + * fcst_thresh?param1=val1;param2=val2[;...] where the params can be the following:
    + *
  • units set to either mm or in (input assumed to be in mm)
  • format set to + * the java formatting string to apply, for example 0.00#
  • symbol set to either letters + * or math, for example ge or >=, respectively
+ * + * @param fcstTag Template tag name (including params) for fcst_thresh + * @param fcstThresh Template map value to be formatted + * @return + */ + public static String formatThresh(final String fcstTag, final String fcstThresh) { + String strThreshRet = fcstThresh; + MVOrderedMap mapParams = parseTagParams(fcstTag); + DecimalFormat format = new DecimalFormat("0.000"); + + // attempt to parse the input threshold + String strSymbol; + String strThresh; + double dblThresh; + Matcher matFcstThresh = MVUtil.thresh.matcher(fcstThresh); + if (matFcstThresh.matches()) { + strSymbol = matFcstThresh.group(1); + strThresh = matFcstThresh.group(2); + dblThresh = Double.parseDouble(strThresh); } else { - continue; + return strThreshRet; } - } - if (Character.digit(s.charAt(i), radix) < 0) { - return false; - } + + // change the units, if requested + if (mapParams.containsKey("units")) { + String strUnits = mapParams.get("units").toString(); + if (strUnits.equals("in")) { + strThresh = format.format(dblThresh /= 25.4); + strThreshRet = strSymbol + strThresh; + } + } + + // change the format, if requested + if (mapParams.containsKey("format")) { + String strFormat = mapParams.get("format").toString(); + strThresh = new DecimalFormat(strFormat).format(dblThresh); + strThreshRet = strSymbol + strThresh; + } + + // change the logic symbol, if requested + if (mapParams.containsKey("symbol")) { + String strSymbolType = mapParams.get("symbol").toString(); + if (strSymbolType.equals("letters")) { + strSymbol = strSymbol.replace("==", "eq") + .replace("!=", "ne") + .replace("<=", "le") + .replace(">=", "ge") + .replace("<", "lt") + .replace(">", "gt"); + strThreshRet = strSymbol + strThresh; + } + } + + return strThreshRet; + } + + public static String buildTemplateString( + final String tmpl, final MVOrderedMap vals, + final PrintStream printStream) throws ValidationException { + return buildTemplateString(tmpl, vals, null, printStream); + } + + /** + * Add the fcst_var and stat names from the dep structure of the input job to the input list of + * tmpl map values. + * + * @param job MVPlotJob whose dep structure will be processed + */ + public static MVOrderedMap addTmplValDep(MVPlotJob job) { + MVOrderedMap mapTmplValsPlot = new MVOrderedMap(job.getTmplVal()); + if (job.getIndyVar() != null) { + mapTmplValsPlot.put("indy_var", job.getIndyVar()); + } + for (int intY = 1; intY <= 2; intY++) { + + // get a list of dep groups + MVOrderedMap[] listDepGroup = job.getDepGroups(); + MVOrderedMap mapDep = null; + if (listDepGroup.length > 0) { + mapDep = (MVOrderedMap) listDepGroup[0].get("dep" + intY); + } + if (mapDep != null) { + Map.Entry[] listDep = mapDep.getOrderedEntries(); + + // build tmpl map values for each fcst_var + String strDep = "dep" + intY; + for (int i = 0; i < listDep.length; i++) { + + // resolve the fcst_var and stats for the current dep + String strFcstVar = listDep[i].getKey().toString(); + String[] listStat = (String[]) listDep[i].getValue(); + + // build and add the fcst_var to the tmpl value map + String strDepFcstVar = strDep + "_" + (i + 1); + mapTmplValsPlot.put(strDepFcstVar, strFcstVar); + for (int j = 0; j < listStat.length; j++) { + mapTmplValsPlot.put(strDepFcstVar + "_stat" + (j + 1), listStat[j]); + } + } + } + } + return mapTmplValsPlot; + } + + /** + * Creates a string representation of an R collection containing the list of values in the input + * list, val. + * + * @param val List of values to print in the R collection + * @param ticks (optional) Print tickmarks around values, for when constituents are factors as + * opposed to numeric, defaults to true + * @return String representation of the R collection + */ + public static String printRCol(final Object[] val, final boolean ticks) { + StringBuilder rStr = new StringBuilder("c("); + for (int i = 0; i < val.length; i++) { + if (0 < i) { + rStr.append(", "); + } + String value = val[i].toString().replace("&", "&").replace(">", ">") + .replace("<", "<"); + if (ticks) { + rStr.append("\"").append(value).append("\""); + } else { + rStr.append(value); + } + } + rStr.append(")"); + return rStr.toString(); + } + + /** + * Creates a string representation of an R collection containing the list of values in the input + * list, val. + * + * @param val List of values to print in the R collection + * @return String representation of the R collection + */ + public static String[] printYamlCol(final Object[] val) { + List result = new ArrayList<>(); + for (Object o : val) { + String value = o.toString().replace("&", "&").replace(">", ">") + .replace("<", "<"); + result.add(value); + } + return result.toArray(new String[0]); + } + + public static String printRCol(final Object[] val) { + return printRCol(val, true); + } + + /** + * Parses an R collection string representation and returns a list of the values in the + * collection. An R collection has syntax c(1, 2, 3) or c("a", "bb", "ccc") + * + * @param strRCol + * @return list of String representations of each collection member + */ + public static String[] parseRCol(final String strRCol) { + if (strRCol.contains("\"")) { + Matcher matRColStr = Pattern.compile("c\\(\\s*\"(.*)\"\\s*\\)").matcher(strRCol); + if (!matRColStr.matches()) { + return new String[]{}; + } + String strList = matRColStr.group(1); + List list = new ArrayList<>(); + while (strList.matches(".*\"\\s*,\\s*\".*")) { + list.add(strList.replaceFirst("\"\\s*,\\s*\".*", "")); + strList = strList.replaceFirst(".*?\"\\s*,\\s*\"", ""); + } + list.add(strList.replaceFirst("\"\\s*,\\s*\".*", "")); + return list.toArray(new String[list.size()]); + } else { + Matcher matRColNum = Pattern.compile("c\\(\\s*(.*)\\s*\\)").matcher(strRCol); + if (!matRColNum.matches()) { + return new String[]{}; + } + String strList = matRColNum.group(1); + return strList.split("\\s*,\\s*"); + } + } + + private static void populateInt(Map tableRTags, String tagName, String strValue, int def) { + int valInt = def; + try { + valInt = Integer.parseInt(strValue); + } catch (Exception e) { + } + tableRTags.put(tagName, valInt); + } + + private static void populateDouble(Map tableRTags, String tagName, String strValue, double def) { + double valDouble = def; + try { + valDouble = Double.parseDouble(strValue); + } catch (Exception e) { + } + tableRTags.put(tagName, valDouble); + } + + + /** + * Populate the input table with the plot formatting tag values stored in the input job. + * + * @param tableRTags template value table to receive plot formatting values + * @param job source for plot formatting values + */ + public static void populatePlotFmtTmplYaml(Map tableRTags, MVPlotJob job) { + populateDouble(tableRTags, "alpha", job.getCIAlpha(), 0.05); + + tableRTags.put("plot_type", job.getPlotType()); + populateDouble(tableRTags, "plot_width", job.getPlotWidth(), 11); + populateDouble(tableRTags, "plot_height", job.getPlotHeight(), 8.5); + populateInt(tableRTags, "plot_res", job.getPlotRes(), 72); + + tableRTags.put("plot_units", job.getPlotUnits()); + + String[] valArr = job.getMar().replace("c(", "").replace(")", "").split(","); + List vals = new ArrayList<>(); + try { + for (String s : valArr) { + vals.add(Double.parseDouble(s)); + } + } catch (Exception e) { + } + tableRTags.put("mar", vals); + + valArr = job.getMgp().replace("c(", "").replace(")", "").split(","); + vals = new ArrayList<>(); + try { + for (String s : valArr) { + vals.add(Double.parseDouble(s)); + } + } catch (Exception e) { + } + tableRTags.put("mgp", vals); + + populateInt(tableRTags, "cex", job.getCex(), 1); + populateDouble(tableRTags, "title_weight", job.getTitleWeight(), 1.4); + populateDouble(tableRTags, "title_size", job.getTitleSize(), 1.4); + populateInt(tableRTags, "title_offset", job.getTitleOffset(), -2); + populateDouble(tableRTags, "title_align", job.getTitleAlign(), 0.5); + + populateInt(tableRTags, "xtlab_orient", job.getXtlabOrient(), 1); + populateDouble(tableRTags, "xtlab_perp", job.getXtlabPerp(), 0.75); + populateDouble(tableRTags, "xtlab_horiz", job.getXtlabHoriz(), 0.5); + populateInt(tableRTags, "xtlab_decim", job.getXtlabFreq(), 1); + populateInt(tableRTags, "xtlab_size", job.getXtlabSize(), 1); + + populateInt(tableRTags, "xlab_weight", job.getXlabWeight(), 1); + populateInt(tableRTags, "xlab_size", job.getXlabSize(), 1); + populateInt(tableRTags, "xlab_offset", job.getXlabOffset(), 2); + populateDouble(tableRTags, "xtlab_horiz", job.getXtlabHoriz(), 0.5); + populateDouble(tableRTags, "xlab_align", job.getXlabAlign(), 0.5); + + populateInt(tableRTags, "ytlab_orient", job.getYtlabOrient(), 1); + populateDouble(tableRTags, "ytlab_perp", job.getYtlabPerp(), 0.5); + populateDouble(tableRTags, "ytlab_horiz", job.getYtlabHoriz(), 0.5); + populateInt(tableRTags, "ytlab_size", job.getYtlabSize(), 1); + populateInt(tableRTags, "ylab_weight", job.getYlabWeight(), 1); + populateInt(tableRTags, "ylab_size", job.getYlabSize(), 1); + populateInt(tableRTags, "ylab_offset", job.getYlabOffset(), -2); + populateDouble(tableRTags, "ylab_align", job.getYlabAlign(), 0.5); + + populateInt(tableRTags, "grid_lty", job.getGridLty(), 3); + tableRTags.put("grid_col", job.getGridCol()); + populateInt(tableRTags, "grid_lwd", job.getGridLwd(), 1); + tableRTags.put("grid_x", job.getGridX()); + + populateInt(tableRTags, "x2tlab_orient", job.getX2tlabOrient(), 1); + populateInt(tableRTags, "x2tlab_perp", job.getX2tlabPerp(), 1); + populateDouble(tableRTags, "x2tlab_horiz", job.getX2tlabHoriz(), 0.5); + populateDouble(tableRTags, "x2tlab_size", job.getX2tlabSize(), 0.8); + populateInt(tableRTags, "x2lab_weight", job.getX2labWeight(), 1); + populateDouble(tableRTags, "x2lab_size", job.getX2labSize(), 0.8); + populateDouble(tableRTags, "x2lab_offset", job.getX2labOffset(), -0.5); + populateDouble(tableRTags, "x2lab_align", job.getX2labAlign(), 0.5); + populateInt(tableRTags, "y2tlab_orient", job.getY2tlabOrient(), 1); + populateInt(tableRTags, "y2tlab_perp", job.getY2tlabPerp(), 1); + populateDouble(tableRTags, "y2tlab_horiz", job.getY2tlabHoriz(), 0.5); + populateDouble(tableRTags, "y2tlab_size", job.getY2tlabSize(), 0.5); + populateInt(tableRTags, "y2lab_weight", job.getY2labWeight(), 1); + populateInt(tableRTags, "y2lab_size", job.getY2labSize(), 1); + populateInt(tableRTags, "y2lab_offset", job.getY2labOffset(), 1); + populateDouble(tableRTags, "y2lab_align", job.getY2labAlign(), 0.5); + + populateDouble(tableRTags, "legend_size", job.getLegendSize(), 0.8); + tableRTags.put("legend_box", job.getLegendBox()); + String[] insetStr = job.getLegendInset().replace("c(", "") + .replace(")", "").split(","); + Map insetMap = new HashMap<>(); + try { + insetMap.put("x", Double.valueOf(insetStr[0])); + insetMap.put("y", Double.valueOf(insetStr[1])); + } catch (Exception e) { + insetMap = new HashMap<>(); + } + if (!insetMap.isEmpty()) { + tableRTags.put("legend_inset", insetMap); + } + populateInt(tableRTags, "legend_ncol", job.getLegendNcol(), 3); + populateInt(tableRTags, "caption_weight", job.getCaptionWeight(), 1); + + tableRTags.put("caption_col", job.getCaptionCol()); + populateDouble(tableRTags, "caption_size", job.getCaptionSize(), 0.8); + populateDouble(tableRTags, "caption_offset", job.getCaptionOffset(), 3); + populateDouble(tableRTags, "caption_align", job.getCaptionAlign(), 0); + + tableRTags.put("box_pts", job.getBoxPts().equalsIgnoreCase("true") ? "True" : "False"); + tableRTags.put("box_outline", job.getBoxOutline().equalsIgnoreCase("true") ? "True" : "False"); + populateDouble(tableRTags, "box_boxwex", job.getBoxBoxwex(), 0.2); + tableRTags.put("box_notch", job.getBoxNotch().equalsIgnoreCase("true") ? "True" : "False"); + tableRTags.put("box_avg", job.getBoxAvg().equalsIgnoreCase("true") ? "True" : "False"); + + } + + + /** + * Populate the input table with the plot formatting tag values stored in the input job. + * + * @param tableRTags template value table to receive plot formatting values + * @param job source for plot formatting values + */ + public static void populatePlotFmtTmpl(Map tableRTags, MVPlotJob job) { + tableRTags.put("plot_type", job.getPlotType()); + tableRTags.put("plot_width", job.getPlotWidth()); + tableRTags.put("plot_height", job.getPlotHeight()); + tableRTags.put("plot_res", job.getPlotRes()); + tableRTags.put("plot_units", job.getPlotUnits()); + tableRTags.put("mar", job.getMar()); + tableRTags.put("mgp", job.getMgp()); + tableRTags.put("cex", job.getCex()); + tableRTags.put("title_weight", job.getTitleWeight()); + tableRTags.put("title_size", job.getTitleSize()); + tableRTags.put("title_offset", job.getTitleOffset()); + tableRTags.put("title_align", job.getTitleAlign()); + tableRTags.put("xtlab_orient", job.getXtlabOrient()); + tableRTags.put("xtlab_perp", job.getXtlabPerp()); + tableRTags.put("xtlab_horiz", job.getXtlabHoriz()); + tableRTags.put("xtlab_decim", job.getXtlabFreq()); + tableRTags.put("xtlab_size", job.getXtlabSize()); + tableRTags.put("xlab_weight", job.getXlabWeight()); + tableRTags.put("xlab_size", job.getXlabSize()); + tableRTags.put("xlab_offset", job.getXlabOffset()); + tableRTags.put("xlab_align", job.getXlabAlign()); + tableRTags.put("ytlab_orient", job.getYtlabOrient()); + tableRTags.put("ytlab_perp", job.getYtlabPerp()); + tableRTags.put("ytlab_horiz", job.getYtlabHoriz()); + tableRTags.put("ytlab_size", job.getYtlabSize()); + tableRTags.put("ylab_weight", job.getYlabWeight()); + tableRTags.put("ylab_size", job.getYlabSize()); + tableRTags.put("ylab_offset", job.getYlabOffset()); + tableRTags.put("ylab_align", job.getYlabAlign()); + tableRTags.put("grid_lty", job.getGridLty()); + tableRTags.put("grid_col", job.getGridCol()); + tableRTags.put("grid_lwd", job.getGridLwd()); + tableRTags.put("grid_x", job.getGridX()); + tableRTags.put("x2tlab_orient", job.getX2tlabOrient()); + tableRTags.put("x2tlab_perp", job.getX2tlabPerp()); + tableRTags.put("x2tlab_horiz", job.getX2tlabHoriz()); + tableRTags.put("x2tlab_size", job.getX2tlabSize()); + tableRTags.put("x2lab_weight", job.getX2labWeight()); + tableRTags.put("x2lab_size", job.getX2labSize()); + tableRTags.put("x2lab_offset", job.getX2labOffset()); + tableRTags.put("x2lab_align", job.getX2labAlign()); + tableRTags.put("y2tlab_orient", job.getY2tlabOrient()); + tableRTags.put("y2tlab_perp", job.getY2tlabPerp()); + tableRTags.put("y2tlab_horiz", job.getY2tlabHoriz()); + tableRTags.put("y2tlab_size", job.getY2tlabSize()); + tableRTags.put("y2lab_weight", job.getY2labWeight()); + tableRTags.put("y2lab_size", job.getY2labSize()); + tableRTags.put("y2lab_offset", job.getY2labOffset()); + tableRTags.put("y2lab_align", job.getY2labAlign()); + tableRTags.put("legend_size", job.getLegendSize()); + tableRTags.put("legend_box", job.getLegendBox()); + tableRTags.put("legend_inset", job.getLegendInset()); + tableRTags.put("legend_ncol", job.getLegendNcol()); + tableRTags.put("caption_weight", job.getCaptionWeight()); + tableRTags.put("caption_col", job.getCaptionCol()); + tableRTags.put("caption_size", job.getCaptionSize()); + tableRTags.put("caption_offset", job.getCaptionOffset()); + tableRTags.put("caption_align", job.getCaptionAlign()); + tableRTags.put("box_pts", job.getBoxPts()); + tableRTags.put("box_outline", job.getBoxOutline()); + tableRTags.put("box_boxwex", job.getBoxBoxwex()); + tableRTags.put("box_notch", job.getBoxNotch()); + tableRTags.put("box_avg", job.getBoxAvg()); + tableRTags.put("rely_event_hist", job.getRelyEventHist()); + tableRTags.put("ci_alpha", job.getCIAlpha()); + + } + + /** + * Creates a list of length rep of copies of the input val. Mimics the R function of the same + * name + * + * @param val Value to repeat + * @param rep Number of time to repeat + * @return List of repeated values, with length specified by input rep + */ + public static String[] rep(final String val, final int rep) { + if (1 > rep) { + return new String[]{}; + } + String[] listRet = new String[rep]; + for (int i = 0; i < rep; i++) { + listRet[i] = val; + } + return listRet; + } + + /** + * Creates a list of length rep of copies of the input val. Mimics the R function of the same + * name + * + * @param val Value to repeat + * @param rep Number of time to repeat + * @return List of repeated values, with length specified by input rep + */ + public static Integer[] rep(final int val, final int rep) { + if (1 > rep) { + return new Integer[]{}; + } + Integer[] listRet = new Integer[rep]; + for (int i = 0; i < rep; i++) { + listRet[i] = val; + } + return listRet; + } + + /** + * Creates a list of integers where the first element is min, the second is min+1 .... the last is + * max name + * + * @param min The first value + * @param max The last number + * @return List of values + */ + public static Integer[] repPlusOne(final int min, final int max) { + + Integer[] listRet = new Integer[max - min + 1]; + int start = min; + for (int i = 0; i < (max - min + 1); i++) { + listRet[i] = start; + start++; + } + return listRet; } - return true; - } - - public static class NameFilter implements FilenameFilter { - private String currentName; - - NameFilter(String name) { - this.currentName = name; - } - - @Override - public boolean accept(File dir, String name) { - return name.contains(currentName) && name.endsWith("xml"); - } + public static boolean isValidLineType(final String lineType) { + boolean result = false; + for (String type : lineTypes) { + if (type.equalsIgnoreCase(lineType)) { + result = true; + break; + } + } + return result; + } - } - - public static String domSourceToString(final Document document) { - String result = ""; - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - - DOMSource source = new DOMSource(document); - try (StringWriter stringWriter = new StringWriter()) { - transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); - Transformer transformer = transformerFactory.newTransformer(); - transformer.transform(source, new StreamResult(stringWriter)); - result = stringWriter.toString(); - } catch (TransformerException | IOException e) { - logger.error( e.getMessage()); - } - return result; - } - - public static Document createDocument() throws ParserConfigurationException { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - DocumentBuilder dBuilder = dbf.newDocumentBuilder(); - return dBuilder.newDocument(); - } - - public static void safeClose(FileWriter fileWriter) { - if (fileWriter != null) { - try { - fileWriter.close(); - } catch (IOException e) { - logger.error( e.getMessage()); - } - } - } - - public static String cleanString(String aString) { - if (aString == null) { - return null; - } - StringBuilder cleanString = new StringBuilder(); - for (int i = 0; i < aString.length(); ++i) { - cleanString.append(cleanChar(aString.charAt(i))); - } - return cleanString.toString(); - } - - private static char cleanChar(char aChar) { - - // 0 - 9 - for (int i = 48; i < 58; ++i) { - if (aChar == i) { - return (char) i; - } - } - - // 'A' - 'Z' - for (int i = 65; i < 91; ++i) { - if (aChar == i) { - return (char) i; - } - } - - // 'a' - 'z' - for (int i = 97; i < 123; ++i) { - if (aChar == i) { - return (char) i; - } - } - - // other valid characters - switch (aChar) { - case '/': - return '/'; - case '.': - return '.'; - case '-': - return '-'; - case '_': - return '_'; - case ' ': - return ' '; - case '(': - return '('; - case ')': - return ')'; - } - return '%'; - } - - public static synchronized String formatPlotFormat(Date date) { - return PLOT_FORMAT.format(date); - } - - public static synchronized Date parsePlotFormat(String dateStr) throws ParseException { - return PLOT_FORMAT.parse(dateStr); - } - - public static synchronized String formatPerf(double val) { - return DECIMAL_FORMAT.format(val); - } - - public static String[][] getDiffSeriesArr(String diffSeriesTemplate) { - if (diffSeriesTemplate.equals("list()")) { - return new String[][]{}; - } - String diffSeries = diffSeriesTemplate.replace("list(", ""); - String[] diffSeriesArr = diffSeries.split("c\\("); - List result = new ArrayList<>(); - for (String serie : diffSeriesArr) { - if (!serie.isEmpty()) { - serie = serie.replaceAll("\"", "").replaceAll("\\(", "").replaceAll("\\)", ""); - String[] args = serie.split("\\s*,\\s*"); - result.add(args); - } - } - return result.toArray(new String[0][]); - } - - public static synchronized void createYamlFile(final String fileName, final Map info) throws IOException { - DumperOptions options = new DumperOptions(); - //options.setDefaultScalarStyle(DumperOptions.ScalarStyle.SINGLE_QUOTED); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - //sort by key - Map sortedMap = new TreeMap<>(info); - Yaml yaml = new Yaml(options); - try (FileWriter writer = new FileWriter(fileName)) { - yaml.dump(sortedMap, writer); - } - } - public static boolean isEtbJob(MVPlotJob job){ - if (job.getDiffSeries1Count() == 0 && job.getDiffSeries2Count() == 0){ - return false; - } - String[][] diffs = MVUtil.getDiffSeriesArr(job.getDiffSeries1()); - for (String[] diff : diffs){ - if (diff.length > 2 && diff[2].equals("ETB")){ + public static boolean isNumeric(final Object obj) { + try { + Double.parseDouble(String.valueOf(obj)); + } catch (NumberFormatException nfe) { + return false; + } return true; - } } - diffs = MVUtil.getDiffSeriesArr(job.getDiffSeries2()); - for (String[] diff : diffs){ - if (diff.length > 2 && diff[2].equals("ETB")){ + + + public static boolean isInteger(String s, int radix) { + if (s.isEmpty()) { + return false; + } + for (int i = 0; i < s.length(); i++) { + if (i == 0 && s.charAt(i) == '-') { + if (s.length() == 1) { + return false; + } else { + continue; + } + } + if (Character.digit(s.charAt(i), radix) < 0) { + return false; + } + } return true; - } - } - return false; - } - - /** - * Reads the version of METviewer from MANIFEST.MF file - * @return version number - */ - public static String getVersionNumber() { - String version = ""; - String className = MVUtil.class.getSimpleName() + ".class"; - String classPath = MVUtil.class.getResource(className).toString(); - if (classPath.startsWith("jar")) { - // Class from JAR - String manifestPath = classPath.substring(0, - classPath.lastIndexOf("!") + 1) + "/META-INF/MANIFEST.MF"; - URL url = null; - try { - url = new URL(manifestPath); - } catch (MalformedURLException e) { - // Do nothing - } - if (url != null) { - try (InputStream inputStream = url.openStream()) { - Manifest manifest = new Manifest(inputStream); - Attributes attr = manifest.getMainAttributes(); - version = attr.getValue("Specification-Version"); - } catch (IOException e) { - //do nothing + } + + public static class NameFilter implements FilenameFilter { + + private String currentName; + + NameFilter(String name) { + this.currentName = name; + } + + @Override + public boolean accept(File dir, String name) { + return name.contains(currentName) && name.endsWith("xml"); + } + + } + + public static String domSourceToString(final Document document) { + String result = ""; + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + + DOMSource source = new DOMSource(document); + try (StringWriter stringWriter = new StringWriter()) { + transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); + Transformer transformer = transformerFactory.newTransformer(); + transformer.transform(source, new StreamResult(stringWriter)); + result = stringWriter.toString(); + } catch (TransformerException | IOException e) { + logger.error(e.getMessage()); + } + return result; + } + + public static Document createDocument() throws ParserConfigurationException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder dBuilder = dbf.newDocumentBuilder(); + return dBuilder.newDocument(); + } + + public static void safeClose(FileWriter fileWriter) { + if (fileWriter != null) { + try { + fileWriter.close(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + } + + public static String cleanString(String aString) { + if (aString == null) { + return null; + } + StringBuilder cleanString = new StringBuilder(); + for (int i = 0; i < aString.length(); ++i) { + cleanString.append(cleanChar(aString.charAt(i))); + } + return cleanString.toString(); + } + + private static char cleanChar(char aChar) { + + // 0 - 9 + for (int i = 48; i < 58; ++i) { + if (aChar == i) { + return (char) i; + } + } + + // 'A' - 'Z' + for (int i = 65; i < 91; ++i) { + if (aChar == i) { + return (char) i; + } + } + + // 'a' - 'z' + for (int i = 97; i < 123; ++i) { + if (aChar == i) { + return (char) i; + } + } + + // other valid characters + switch (aChar) { + case '/': + return '/'; + case '.': + return '.'; + case '-': + return '-'; + case '_': + return '_'; + case ' ': + return ' '; + case '(': + return '('; + case ')': + return ')'; + } + return '%'; + } + + public static synchronized String formatPlotFormat(Date date) { + return PLOT_FORMAT.format(date); + } + + public static synchronized Date parsePlotFormat(String dateStr) throws ParseException { + return PLOT_FORMAT.parse(dateStr); + } + + public static synchronized String formatPerf(double val) { + return DECIMAL_FORMAT.format(val); + } + + public static String[][] getDiffSeriesArr(String diffSeriesTemplate) { + if (diffSeriesTemplate.equals("list()")) { + return new String[][]{}; + } + String diffSeries = diffSeriesTemplate.replace("list(", ""); + String[] diffSeriesArr = diffSeries.split("c\\("); + List result = new ArrayList<>(); + for (String serie : diffSeriesArr) { + if (!serie.isEmpty()) { + serie = serie.replaceAll("\"", "").replaceAll("\\(", "").replaceAll("\\)", ""); + String[] args = serie.split("\\s*,\\s*"); + result.add(args); + } + } + return result.toArray(new String[0][]); + } + + public static synchronized void createYamlFile(final String fileName, final Map info) throws IOException { + DumperOptions options = new DumperOptions(); + //options.setDefaultScalarStyle(DumperOptions.ScalarStyle.SINGLE_QUOTED); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + //sort by key + Map sortedMap = new TreeMap<>(info); + Yaml yaml = new Yaml(options); + try (FileWriter writer = new FileWriter(fileName)) { + yaml.dump(sortedMap, writer); + } + } + + public static boolean isEtbJob(MVPlotJob job) { + if (job.getDiffSeries1Count() == 0 && job.getDiffSeries2Count() == 0) { + return false; + } + String[][] diffs = MVUtil.getDiffSeriesArr(job.getDiffSeries1()); + for (String[] diff : diffs) { + if (diff.length > 2 && diff[2].equals("ETB")) { + return true; + } + } + diffs = MVUtil.getDiffSeriesArr(job.getDiffSeries2()); + for (String[] diff : diffs) { + if (diff.length > 2 && diff[2].equals("ETB")) { + return true; + } + } + return false; + } + + /** + * Reads the version of METviewer from MANIFEST.MF file + * + * @return version number + */ + public static String getVersionNumber() { + String version = ""; + String className = MVUtil.class.getSimpleName() + ".class"; + String classPath = MVUtil.class.getResource(className).toString(); + if (classPath.startsWith("jar")) { + // Class from JAR + String manifestPath = classPath.substring(0, + classPath.lastIndexOf("!") + 1) + "/META-INF/MANIFEST.MF"; + URL url = null; + try { + url = new URL(manifestPath); + } catch (MalformedURLException e) { + // Do nothing + } + if (url != null) { + try (InputStream inputStream = url.openStream()) { + Manifest manifest = new Manifest(inputStream); + Attributes attr = manifest.getMainAttributes(); + version = attr.getValue("Specification-Version"); + } catch (IOException e) { + //do nothing + } + } } - } + return version; } - return version; - } } diff --git a/java/edu/ucar/metviewer/rscriptManager/RscriptAggStatManager.java b/java/edu/ucar/metviewer/rscriptManager/RscriptAggStatManager.java index 266a6c55..05fdfe6d 100644 --- a/java/edu/ucar/metviewer/rscriptManager/RscriptAggStatManager.java +++ b/java/edu/ucar/metviewer/rscriptManager/RscriptAggStatManager.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -88,11 +89,12 @@ public void prepareDataFileAndRscript( + mvBatch.getMetCalcpyHome() + "/metcalcpy/agg_stat_event_equalize.py" + " " + eeInfo); - + Map env = new HashMap<>(); + env.put("PYTHONPATH", mvBatch.getMetCalcpyHome()); mvResponse = MVUtil.runRscript(mvBatch.getPython(), mvBatch.getMetCalcpyHome() + "/metcalcpy/agg_stat_event_equalize.py", new String[]{eeInfo}, - new String[]{"PYTHONPATH=" + mvBatch.getMetCalcpyHome()}); + env); }else { @@ -298,7 +300,8 @@ public boolean runPythonScript(MVPlotJob job, Map info) { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - + Map env = new HashMap<>(); + env.put("PYTHONPATH", mvBatch.getMetCalcpyHome()); if(job.isModeJob() || job.isMtdJob()){ mvBatch.print(mvBatch.getPython() + "\n" @@ -309,7 +312,7 @@ public boolean runPythonScript(MVPlotJob job, Map info) { mvResponse = MVUtil.runRscript(mvBatch.getPython(), mvBatch.getMetCalcpyHome() + rScriptFile, new String[]{aggInfo}, - new String[]{"PYTHONPATH=" + mvBatch.getMetCalcpyHome()}); + env); }else { mvBatch.print(mvBatch.getPython() @@ -321,7 +324,7 @@ public boolean runPythonScript(MVPlotJob job, Map info) { mvResponse = MVUtil.runRscript(mvBatch.getPython(), mvBatch.getMetCalcpyHome() + rScriptFile, new String[]{aggInfo}, - new String[]{"PYTHONPATH=" + mvBatch.getMetCalcpyHome()}); + env); } stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { diff --git a/java/edu/ucar/metviewer/rscriptManager/RscriptNoneStatManager.java b/java/edu/ucar/metviewer/rscriptManager/RscriptNoneStatManager.java index d780065d..4ff78153 100644 --- a/java/edu/ucar/metviewer/rscriptManager/RscriptNoneStatManager.java +++ b/java/edu/ucar/metviewer/rscriptManager/RscriptNoneStatManager.java @@ -9,6 +9,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -139,11 +140,15 @@ public boolean runPythonScript(MVPlotJob job, Map info) { + mvBatch.getMetPlotpyHome() + job.getPlotTmpl() + "\n" + configFileName); - mvResponse = MVUtil.runRscript(mvBatch.getPython(), + Map env = new HashMap<>(); + env.put("PYTHONPATH", mvBatch.getMetPlotpyHome() + ":" + mvBatch.getMetCalcpyHome()); + env.put("METPLOTPY_BASE", mvBatch.getMetPlotpyHome()); + env.put("PRE_LOAD_CHROME", "True"); + + mvResponse = MVUtil.runRscript( mvBatch.getPython(), mvBatch.getMetPlotpyHome() + job.getPlotTmpl(), new String[]{configFileName}, - new String[]{"PYTHONPATH=" + mvBatch.getMetPlotpyHome() + ":" + mvBatch.getMetCalcpyHome(), - "METPLOTPY_BASE="+mvBatch.getMetPlotpyHome() }); + env); stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { mvBatch.print(mvResponse.getInfoMessage()); diff --git a/java/edu/ucar/metviewer/rscriptManager/RscriptSumStatManager.java b/java/edu/ucar/metviewer/rscriptManager/RscriptSumStatManager.java index fee6d7cf..a3ed5990 100644 --- a/java/edu/ucar/metviewer/rscriptManager/RscriptSumStatManager.java +++ b/java/edu/ucar/metviewer/rscriptManager/RscriptSumStatManager.java @@ -9,6 +9,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -127,10 +128,12 @@ public boolean runPythonScript(MVPlotJob job, Map info) { + mvBatch.getMetCalcpyHome() + "/metcalcpy/sum_stat.py" + " " + sumInfo); + Map env = new HashMap<>(); + env.put("PYTHONPATH", mvBatch.getMetCalcpyHome()); mvResponse = MVUtil.runRscript(mvBatch.getPython(), mvBatch.getMetCalcpyHome() + "/metcalcpy/sum_stat.py", new String[]{sumInfo}, - new String[]{"PYTHONPATH=" + mvBatch.getMetCalcpyHome()}); + env); stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { mvBatch.print(mvResponse.getInfoMessage()); diff --git a/java/edu/ucar/metviewer/scorecard/rscript/AggPythonManager.java b/java/edu/ucar/metviewer/scorecard/rscript/AggPythonManager.java index 8aa406eb..6c1950c5 100644 --- a/java/edu/ucar/metviewer/scorecard/rscript/AggPythonManager.java +++ b/java/edu/ucar/metviewer/scorecard/rscript/AggPythonManager.java @@ -122,11 +122,12 @@ public void calculateStatsForRow(Map mapRow, String threadName) { stopWatch.start(); logger.info(python + " " + metCalcpyHome + PYTHON_SCRIPT + " " + aggInfoFileName); - + Map env = new HashMap<>(); + env.put("PYTHONPATH", metCalcpyHome); MvResponse mvResponse = MVUtil.runRscript(python, metCalcpyHome + PYTHON_SCRIPT, new String[]{aggInfoFileName}, - new String[]{"PYTHONPATH=" + metCalcpyHome}); + env); stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { logger.info(mvResponse.getInfoMessage()); diff --git a/java/edu/ucar/metviewer/scorecard/rscript/SumPythonManager.java b/java/edu/ucar/metviewer/scorecard/rscript/SumPythonManager.java index 38a7b45e..7eab529e 100644 --- a/java/edu/ucar/metviewer/scorecard/rscript/SumPythonManager.java +++ b/java/edu/ucar/metviewer/scorecard/rscript/SumPythonManager.java @@ -123,11 +123,12 @@ public void calculateStatsForRow(Map mapRow, String threadName) t stopWatch.start(); logger.info(python + " " + metCalcpyHome + PYTHON_SCRIPT + " " + sumInfoFileName); - + Map env = new HashMap<>(); + env.put("PYTHONPATH", metCalcpyHome); MvResponse mvResponse = MVUtil.runRscript(python, metCalcpyHome + PYTHON_SCRIPT, new String[]{sumInfoFileName}, - new String[]{"PYTHONPATH=" + metCalcpyHome}); + env); stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { @@ -236,11 +237,12 @@ public void calculateStatsForRow(Map mapRow, String threadName) t stopWatch.start(); logger.info(python + " " + metCalcpyHome + "/metcalcpy/scorecard.py" + " " + pythonFileInfo); - + Map env = new HashMap<>(); + env.put("PYTHONPATH", metCalcpyHome); MvResponse mvResponse = MVUtil.runRscript(python, metCalcpyHome + "/metcalcpy/scorecard.py", new String[]{pythonFileInfo}, - new String[]{"PYTHONPATH=" + metCalcpyHome}); + env); stopWatch.stop(); if (mvResponse.getInfoMessage() != null) { diff --git a/lib/.DS_Store b/lib/.DS_Store new file mode 100644 index 00000000..5008ddfc Binary files /dev/null and b/lib/.DS_Store differ diff --git a/lib/commons-fileupload-1.6.0.jar b/lib/commons-fileupload-1.6.0.jar deleted file mode 100644 index 63af9e12..00000000 Binary files a/lib/commons-fileupload-1.6.0.jar and /dev/null differ diff --git a/lib/commons-fileupload2-core-2.0.0-M4.jar b/lib/commons-fileupload2-core-2.0.0-M4.jar new file mode 100644 index 00000000..4054cf3f Binary files /dev/null and b/lib/commons-fileupload2-core-2.0.0-M4.jar differ diff --git a/lib/commons-fileupload2-jakarta-servlet5-2.0.0-M4.jar b/lib/commons-fileupload2-jakarta-servlet5-2.0.0-M4.jar new file mode 100644 index 00000000..36cff245 Binary files /dev/null and b/lib/commons-fileupload2-jakarta-servlet5-2.0.0-M4.jar differ diff --git a/lib/commons-lang3-3.11.jar b/lib/commons-lang3-3.11.jar deleted file mode 100644 index bbaa8a61..00000000 Binary files a/lib/commons-lang3-3.11.jar and /dev/null differ diff --git a/lib/commons-lang3-3.19.0.jar b/lib/commons-lang3-3.19.0.jar new file mode 100644 index 00000000..1ee7e036 Binary files /dev/null and b/lib/commons-lang3-3.19.0.jar differ diff --git a/lib/commons-logging-1.1.3.jar b/lib/commons-logging-1.1.3.jar deleted file mode 100644 index ab512540..00000000 Binary files a/lib/commons-logging-1.1.3.jar and /dev/null differ diff --git a/lib/jakarta.servlet-api-5.0.0.jar b/lib/jakarta.servlet-api-5.0.0.jar new file mode 100644 index 00000000..88b788a0 Binary files /dev/null and b/lib/jakarta.servlet-api-5.0.0.jar differ diff --git a/lib/javax.servlet-api-4.0.1.jar b/lib/javax.servlet-api-4.0.1.jar deleted file mode 100644 index 844ec7f1..00000000 Binary files a/lib/javax.servlet-api-4.0.1.jar and /dev/null differ diff --git a/lib/log4j-api-2.17.1.jar b/lib/log4j-api-2.17.1.jar deleted file mode 100644 index 1aae243b..00000000 Binary files a/lib/log4j-api-2.17.1.jar and /dev/null differ diff --git a/lib/log4j-api-2.25.2.jar b/lib/log4j-api-2.25.2.jar new file mode 100644 index 00000000..b6255e68 Binary files /dev/null and b/lib/log4j-api-2.25.2.jar differ diff --git a/lib/log4j-core-2.17.1.jar b/lib/log4j-core-2.17.1.jar deleted file mode 100644 index 46825278..00000000 Binary files a/lib/log4j-core-2.17.1.jar and /dev/null differ diff --git a/lib/log4j-core-2.25.2.jar b/lib/log4j-core-2.25.2.jar new file mode 100644 index 00000000..4113369e Binary files /dev/null and b/lib/log4j-core-2.25.2.jar differ diff --git a/lib/log4j-iostreams-2.17.1.jar b/lib/log4j-iostreams-2.17.1.jar deleted file mode 100644 index 29eb30c0..00000000 Binary files a/lib/log4j-iostreams-2.17.1.jar and /dev/null differ diff --git a/lib/log4j-iostreams-2.25.2.jar b/lib/log4j-iostreams-2.25.2.jar new file mode 100644 index 00000000..dbaee007 Binary files /dev/null and b/lib/log4j-iostreams-2.25.2.jar differ diff --git a/lib/log4j-web-2.17.1.jar b/lib/log4j-web-2.17.1.jar deleted file mode 100644 index a6c085ad..00000000 Binary files a/lib/log4j-web-2.17.1.jar and /dev/null differ diff --git a/lib/log4j-web-2.25.2.jar b/lib/log4j-web-2.25.2.jar new file mode 100644 index 00000000..edbd75fb Binary files /dev/null and b/lib/log4j-web-2.25.2.jar differ diff --git a/lib/mariadb-java-client-2.7.1.jar b/lib/mariadb-java-client-2.7.1.jar deleted file mode 100644 index 691d1747..00000000 Binary files a/lib/mariadb-java-client-2.7.1.jar and /dev/null differ diff --git a/lib/mariadb-java-client-3.5.6.jar b/lib/mariadb-java-client-3.5.6.jar new file mode 100644 index 00000000..447793ce Binary files /dev/null and b/lib/mariadb-java-client-3.5.6.jar differ diff --git a/lib/servlet-api.jar b/lib/servlet-api.jar deleted file mode 100644 index 5c54e852..00000000 Binary files a/lib/servlet-api.jar and /dev/null differ diff --git a/lib/slf4j-api-1.7.5.jar b/lib/slf4j-api-1.7.5.jar deleted file mode 100644 index 8766455d..00000000 Binary files a/lib/slf4j-api-1.7.5.jar and /dev/null differ diff --git a/lib/slf4j-api-2.0.17.jar b/lib/slf4j-api-2.0.17.jar new file mode 100644 index 00000000..26b15455 Binary files /dev/null and b/lib/slf4j-api-2.0.17.jar differ diff --git a/lib/slf4j-log4j12-1.7.5.jar b/lib/slf4j-log4j12-1.7.5.jar deleted file mode 100644 index afce5c21..00000000 Binary files a/lib/slf4j-log4j12-1.7.5.jar and /dev/null differ diff --git a/lib/slf4j-reload4j-2.0.17.jar b/lib/slf4j-reload4j-2.0.17.jar new file mode 100644 index 00000000..3cd24fb3 Binary files /dev/null and b/lib/slf4j-reload4j-2.0.17.jar differ diff --git a/lib/snakeyaml-2.0.jar b/lib/snakeyaml-2.0.jar deleted file mode 100644 index 469b0430..00000000 Binary files a/lib/snakeyaml-2.0.jar and /dev/null differ diff --git a/lib/snakeyaml-2.5.jar b/lib/snakeyaml-2.5.jar new file mode 100644 index 00000000..81864514 Binary files /dev/null and b/lib/snakeyaml-2.5.jar differ diff --git a/lib/tomcat-jdbc-10.1.46.jar b/lib/tomcat-jdbc-10.1.46.jar new file mode 100644 index 00000000..b7ac648f Binary files /dev/null and b/lib/tomcat-jdbc-10.1.46.jar differ diff --git a/lib/tomcat-jdbc-8.5.61.jar b/lib/tomcat-jdbc-8.5.61.jar deleted file mode 100644 index 23236c55..00000000 Binary files a/lib/tomcat-jdbc-8.5.61.jar and /dev/null differ diff --git a/webapp/metviewer/WEB-INF/web.xml b/webapp/metviewer/WEB-INF/web.xml index c2b41810..a93d464d 100644 --- a/webapp/metviewer/WEB-INF/web.xml +++ b/webapp/metviewer/WEB-INF/web.xml @@ -1,8 +1,8 @@ - - + METviewer METviewer