Skip to content

Commit

Permalink
v1.78
Browse files Browse the repository at this point in the history
  • Loading branch information
Bob Simons committed May 30, 2017
1 parent cf41ac0 commit e34fa3b
Show file tree
Hide file tree
Showing 53 changed files with 1,029 additions and 194 deletions.
7 changes: 5 additions & 2 deletions WEB-INF/classes/com/cohort/array/Attributes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,8 @@ public String toString() {

/**
* This removes any entry which has a String value of 'value'.
* value must equal the pa.toString() of the attribute's value,
* so a string with the word null must be requesth here as "\"null\""
*
* @param value Any attribute that has this value (when evaluated as a String)
* will be removed.
Expand All @@ -623,6 +625,7 @@ public void removeValue(String value) {
Iterator it = hashmap.keySet().iterator(); //iterator (not enumeration) since I use it.remove() below
while (it.hasNext()) {
String name = (String)it.next();
//String2.log(">> lookFor=" + value + " found=" + get(name).toString());
if (get(name).toString().equals(value))
it.remove();
}
Expand Down Expand Up @@ -823,7 +826,7 @@ public void fromNccsvStrings() {
/**
* This makes a set of addAttributes which are needed to change a into b.
* If an attribute in 'a' needs to be set to null, this sets it to the String
* "null" instead of just nulling it.
* "null" instead of just nulling it.
*
* @param a an Attributes object
* @param b another Attributes object
Expand Down Expand Up @@ -1115,7 +1118,7 @@ public static void test() throws Exception {
" number=11, 22\n",
"atts=\n" + atts.toString());
a.add(atts);
a.removeValue("null");
a.removeValue("\"null\"");
Test.ensureEqual(a, b, "");
Test.ensureEqual(a.toString(), b.toString(), "");

Expand Down
2 changes: 1 addition & 1 deletion WEB-INF/classes/com/cohort/array/StringArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ public StringArray toJson() {
*/
public void fromJson() {
for (int i = 0; i < size; i++)
array[i] = String2.canonical(String2.fromJson(array[i])); //doesn't require enclosing "'s
array[i] = String2.canonical(String2.fromJsonNotNull(array[i])); //doesn't require enclosing "'s
}

/**
Expand Down
34 changes: 30 additions & 4 deletions WEB-INF/classes/com/cohort/util/String2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2067,19 +2067,26 @@ else if (ch == '\b') {} //remove it
return sb.toString();
}


/**
* This is like the other fromJson, but returns "" instead of null.
*/
public static String fromJsonNotNull(String s) {
s = fromJson(s);
return s == null? "" : s;
}

/**
* This returns the unJSON version of a JSON string
* (surrounding "'s (if any) are removed and \\, \f, \n, \r, \t, \/, and \" are unescaped).
* This is very liberal in what it accepts, including all common C escaped characters:
* http://msdn.microsoft.com/en-us/library/h21280bw%28v=vs.80%29.aspx
* null and "null" are returned as null.
* "null" returns the String "null". null returns null.
*
* @param s it may be enclosed by "'s, or not.
* @return the decoded string
*/
public static String fromJson(String s) {
if (s == null || s.equals("null"))
if (s == null)
return null;
if (s.length() >= 2 && s.charAt(0) == '"' && s.charAt(s.length() - 1) == '"')
s = s.substring(1, s.length() - 1);
Expand Down Expand Up @@ -2227,7 +2234,8 @@ public static String toNccsvDataString(String s) {
if (s.startsWith(" ") ||
s.endsWith(" ") ||
s.indexOf(',') >= 0 ||
s.indexOf('"') >= 0)
s.indexOf('"') >= 0 ||
s.equals("null"))
return "\"" + sb.toString() + "\"";
return sb.toString();
}
Expand All @@ -2248,6 +2256,7 @@ public static String toNccsvAttString(String s) {
s.endsWith(" ") ||
s.indexOf(',') >= 0 ||
s.indexOf('"') >= 0 ||
s.equals("null") ||
NCCSV_CHAR_ATT_PATTERN.matcher(s).matches() || //Looks Like A char
NCCSV_LLA_NUMBER_PATTERN.matcher(s).matches()) //Looks Like A number
return "\"" + sb.toString() + "\"";
Expand Down Expand Up @@ -6133,5 +6142,22 @@ public static String guessAcddContactType(String name) {
return null;
}

/**
* Convert plain text to simple regex by backslash encoding
* all special regex chars.
*/
public static String plainTextToRegex(String s) {
int n = s.length();
StringBuilder sb = new StringBuilder(n + 32);
for (int i = 0; i < n; i++) {
char ch = s.charAt(i);
if ("\\^$.?*+|{}[]()".indexOf(ch) >= 0) // , -
sb.append('\\');
sb.append(ch);
}
return sb.toString();
}



} //End of String2 class.
5 changes: 5 additions & 0 deletions WEB-INF/classes/com/cohort/util/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1459,6 +1459,11 @@ public static void testString2() throws Exception {
Test.ensureEqual(String2.fromJson("\\x6m"), "", "");
Test.ensureEqual(String2.fromJson("\\u006m"), "", "");

Test.ensureEqual(String2.fromJson("null"), "null", "");
Test.ensureEqual(String2.fromJson(null), null, "");
Test.ensureEqual(String2.fromJsonNotNull("null"), "null", "");
Test.ensureEqual(String2.fromJsonNotNull(null), "", "");

//toIso88591String
String2.log("test toIso88591String");
s = String2.annotatedString(String2.toIso88591String("\u0000\n\r\t\f aA\u0091\u00fc\u20ac"));
Expand Down
8 changes: 7 additions & 1 deletion WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public static void main(String args[]) throws Throwable {
// "testNccsvScalar",
// "srtm15plus",
// "ChukchiSea_454a_037a_fcf4", //for Kevin, SocketException: Connection reset
// "ncdcOisst2AmsrAgg",
// "nwioosCoral",
// "-verbose"});

/* if (false) { //one time fixup of scrippsGliders
Expand Down Expand Up @@ -739,6 +739,7 @@ public static void main(String args[]) throws Throwable {
//17336 has data url! //<data_set type="CSV Files">
// "/u00/data/points/inportXml/NOAA/NMFS/AFSC/inport/xml/17336.xml", ".*", ".*", "/u00/data/points/inportData/afsc/", 0, ""));

// "https://inport.nmfs.noaa.gov/inport/item/12866/inport-xml", ".*", ".*", "/u00/data/points/inportData/test/"));
// "https://inport.nmfs.noaa.gov/inport-metadata/NOAA/NMFS/AFSC/inport/xml/17336.xml", ".*", ".*", "/u00/data/points/inportData/test/", 1, ""));
// "/u00/data/points/inportXml/NOAA/NMFS/AFSC/inport/xml/17336.xml", ".*", ".*", "/u00/data/points/inportData/afsc/", 1, ""));
// "/u00/data/points/inportXml/NOAA/NMFS/AFSC/inport/xml/17336.xml", ".*", ".*", "/u00/data/points/inportData/afsc/"));
Expand All @@ -751,6 +752,10 @@ public static void main(String args[]) throws Throwable {
// EDDTableFromAsciiFiles.testGenerateDatasetsXmlFromInPort2();
// EDDTableFromAsciiFiles.testTimeRange2();
// EDD.generateInPortXmlFilesForCoastwatchErddap();
// String2.log(EDDTableFromAsciiFiles.generateDatasetsXmlFromBCODMO(true,
// "https://www.bco-dmo.org/erddap/datasets", "/u00/data/points/bcodmo/",
// "(549122)"));


// EDDTableFromAsciiFiles.testBasic2();
// EDDTableFromAsciiFiles.testTimeZone();
Expand Down Expand Up @@ -1176,6 +1181,7 @@ public static void main(String args[]) throws Throwable {
// EDDTableFromNccsvFiles.test();
// EDDTableFromNccsvFiles.testBasic(true);
// EDDTableFromNccsvFiles.testChar();
// EDDTableFromNccsvFiles.testActualRange();
// EDDGridFromNcFiles.testNccsv();
// EDDTableFromDapSequence.testGenerateDatasetsXml2();
// EDDTableFromErddap.test();
Expand Down
Binary file modified ...NF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc
View file
Binary file not shown.
70 changes: 35 additions & 35 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/NdbcMetStation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ public class NdbcMetStation {
* and near real time data (less quality controlled).
* This changes every month when I get the latest historical data.
*/
public static String firstNearRealTimeData = "2017-04-01T00:00:00";
public static String firstNearRealTimeData = "2017-05-01T00:00:00";
/** Change current year ~Feb 28 when Jan historical files become available. */
public static String HISTORICAL_FILES_CURRENT_YEAR = "2017";

Expand Down Expand Up @@ -2442,27 +2442,27 @@ public static void test46088(Table table) throws Exception {
// http://www.ndbc.noaa.gov/data/realtime2/46088.txt //45 day //top line has precedence
//#YY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD PRES ATMP WTMP DEWP VIS PTDY TIDE
//#yr mo dy hr mn degT m/s m/s m sec sec degT hPa degC degC degC mi hPa ft
//2017 04 01 01 20 270 1.0 1.0 0.1 MM 4.4 MM 1024.6 8.6 8.8 5.5 MM MM MM
//2017 04 01 00 50 300 2.0 2.0 0.2 MM 5.0 MM 1025.3 8.7 8.8 5.5 MM -2.3 MM
seconds = Calendar2.isoStringToEpochSeconds("2017-04-01T01"); //50 min rounds to next hour; usually test 01T01
//2017 05 01 01 20 270 11.0 13.0 0.9 4 3.4 292 1025.3 10.4 9.1 4.9 MM MM MM
//2017 05 01 00 50 270 11.0 13.0 1.0 4 3.6 291 1025.2 10.3 9.1 5.3 MM +0.0 MM
seconds = Calendar2.isoStringToEpochSeconds("2017-05-01T01"); //50 min rounds to next hour; usually test 01T01
row = table.getColumn(timeIndex).indexOf("" + seconds, 0);
Test.ensureEqual(table.getStringData(idIndex, row), "46088", "");
Test.ensureEqual(table.getFloatData(latIndex, row), 48.333f, "");
Test.ensureEqual(table.getFloatData(lonIndex, row), -123.167f, "");
Test.ensureEqual(table.getDoubleData(depthIndex, row), 0, "");
Test.ensureEqual(table.getDoubleData(wdIndex, row), 270, "");
Test.ensureEqual(table.getFloatData(wspdIndex, row), 1f, "");
Test.ensureEqual(table.getFloatData(gstIndex, row), 1f, "");
Test.ensureEqual(table.getFloatData(wvhtIndex, row), .1f, "");
Test.ensureEqual(table.getFloatData(dpdIndex, row), Float.NaN, "");
Test.ensureEqual(table.getFloatData(apdIndex, row), 4.4f, "");
Test.ensureEqual(table.getFloatData(mwdIndex, row), Float.NaN, "");
Test.ensureEqual(table.getFloatData(aprsIndex, row), 1024.6f, "");
Test.ensureEqual(table.getFloatData(atmpIndex, row), 8.6f, "");
Test.ensureEqual(table.getFloatData(wtmpIndex, row), 8.8f, "");
Test.ensureEqual(table.getFloatData(dewpIndex, row), 5.5f, "");
Test.ensureEqual(table.getFloatData(wspdIndex, row), 11f, "");
Test.ensureEqual(table.getFloatData(gstIndex, row), 13f, "");
Test.ensureEqual(table.getFloatData(wvhtIndex, row), .9f, "");
Test.ensureEqual(table.getFloatData(dpdIndex, row), 4f, "");
Test.ensureEqual(table.getFloatData(apdIndex, row), 3.4f, "");
Test.ensureEqual(table.getFloatData(mwdIndex, row), 292f, "");
Test.ensureEqual(table.getFloatData(aprsIndex, row), 1025.3f, "");
Test.ensureEqual(table.getFloatData(atmpIndex, row), 10.4f, "");
Test.ensureEqual(table.getFloatData(wtmpIndex, row), 9.1f, "");
Test.ensureEqual(table.getFloatData(dewpIndex, row), 4.9f, "");
Test.ensureEqual(table.getFloatData(visIndex, row), Float.NaN, ""); //(float)Math2.roundTo(18.5 * Math2.kmPerMile, decimalDigits[visIndex]), "");
Test.ensureEqual(table.getFloatData(ptdyIndex, row), -2.3f, "");
Test.ensureEqual(table.getFloatData(ptdyIndex, row), 0f, "");
Test.ensureEqual(table.getFloatData(tideIndex, row), Float.NaN, ""); //(float)Math2.roundTo(3.0 * Math2.meterPerFoot, decimalDigits[tideIndex]), "");

String2.log("test46088 was successful");
Expand Down Expand Up @@ -2497,28 +2497,28 @@ public static void test46088AddLastNDays(Table table) throws Exception {
//top row has precedence, but not if file already had lower row of data
//#YY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD PRES ATMP WTMP DEWP VIS PTDY TIDE
//#yr mo dy hr mn degT m/s m/s m sec sec degT hPa degC degC degC mi hPa ft
//2017 04 29 02 20 240 9.0 11.0 0.3 2 2.7 87 1026.1 10.2 9.5 6.3 MM MM MM
//2017 04 29 01 50 230 9.0 10.0 0.3 11 3.2 358 1026.1 10.2 9.5 6.2 MM -0.9 MM
double seconds = Calendar2.isoStringToEpochSeconds("2017-04-29T02"); //rounded
//2017 05 25 22 20 210 3.0 4.0 0.5 6 5.2 260 1011.2 11.9 11.1 9.7 MM MM MM
//2017 05 25 21 50 230 3.0 3.0 0.6 8 6.0 271 1011.3 11.7 10.8 9.8 MM -1.3 MM
double seconds = Calendar2.isoStringToEpochSeconds("2017-05-25T22"); //rounded
int row = table.getColumn(timeIndex).indexOf("" + seconds, 0);
Test.ensureTrue(row >= 0, "row=" + row);
Test.ensureEqual(table.getStringData(idIndex, row), "46088", "");
Test.ensureEqual(table.getFloatData(latIndex, row), 48.333f, "");
Test.ensureEqual(table.getFloatData(lonIndex, row), -123.167f, "");
Test.ensureEqual(table.getDoubleData(depthIndex, row), 0, "");
Test.ensureEqual(table.getFloatData(wdIndex, row), 240, "");
Test.ensureEqual(table.getFloatData(wspdIndex, row), 9f, "");
Test.ensureEqual(table.getFloatData(gstIndex, row), 11f, "");
Test.ensureEqual(table.getFloatData(wvhtIndex, row), 0.3f, "");
Test.ensureEqual(table.getFloatData(dpdIndex, row), 2f, "");
Test.ensureEqual(table.getFloatData(apdIndex, row), 2.7f, "");
Test.ensureEqual(table.getFloatData(mwdIndex, row), 87f, "");
Test.ensureEqual(table.getFloatData(aprsIndex, row), 1026.1f, "");
Test.ensureEqual(table.getFloatData(atmpIndex, row), 10.2f, "");
Test.ensureEqual(table.getFloatData(wtmpIndex, row), 9.5f, "");
Test.ensureEqual(table.getFloatData(dewpIndex, row), 6.3f, "");
Test.ensureEqual(table.getFloatData(wdIndex, row), 230, "");
Test.ensureEqual(table.getFloatData(wspdIndex, row), 3f, "");
Test.ensureEqual(table.getFloatData(gstIndex, row), 3f, "");
Test.ensureEqual(table.getFloatData(wvhtIndex, row), 0.6f, "");
Test.ensureEqual(table.getFloatData(dpdIndex, row), 8f, "");
Test.ensureEqual(table.getFloatData(apdIndex, row), 6f, "");
Test.ensureEqual(table.getFloatData(mwdIndex, row), 271, "");
Test.ensureEqual(table.getFloatData(aprsIndex, row), 1011.3f, "");
Test.ensureEqual(table.getFloatData(atmpIndex, row), 11.7f, "");
Test.ensureEqual(table.getFloatData(wtmpIndex, row), 10.8f, "");
Test.ensureEqual(table.getFloatData(dewpIndex, row), 9.8f, "");
Test.ensureEqual(table.getFloatData(visIndex, row), Float.NaN, ""); //(float)Math2.roundTo(18.5 * Math2.kmPerMile, decimalDigits[visIndex]), "");
Test.ensureEqual(table.getFloatData(ptdyIndex, row), -.9f, "");
Test.ensureEqual(table.getFloatData(ptdyIndex, row), -1.3f, "");
Test.ensureEqual(table.getFloatData(tideIndex, row), Float.NaN, "");//(float)Math2.roundTo(3.0 * Math2.meterPerFoot, decimalDigits[tideIndex]), "");

String2.log("test46088AddLastNDays was successful");
Expand Down Expand Up @@ -2786,11 +2786,11 @@ public static void main(String args[]) throws Exception {
//historical monthly files are from: http://www.ndbc.noaa.gov/data/stdmet/<month3Letter>/ e.g., Jan
//!!!!**** Windows GUI My Computer doesn't show all the files in the directory!
// Use DOS window "dir" or Linux ls instead of the GUI.
//downloadNewHistoricalTxtFiles(ndbcHistoricalTxtDir); //time varies, last done 2017-04-28
//downloadNewHistoricalTxtFiles(ndbcHistoricalTxtDir); //time varies, last done 2017-05-25

// 3) *** get latest 45 day files
//DON'T download45DayTextFiles after 45 days after last historicalTxt date.
//download45DayTxtFiles(ndbc45DayTxtDir); //15-30 minutes, last done 2017-04-28
//download45DayTxtFiles(ndbc45DayTxtDir); //15-30 minutes, last done 2017-05-25

// 4) *** Make the nc files
//!!!!**** EACH MONTH, SOME TESTS NEED UPDATING: SEE "UPDATE_EACH_MONTH"
Expand All @@ -2799,7 +2799,7 @@ public static void main(String args[]) throws Exception {
boolean testMode = false; //I used to always run 'true' then run 'false', but not usually now
String ignoreStationsBefore = " "; //use " " to process all stations or lowercase characters to start in middle
//makeSeparateNcFiles(ndbcStationHtmlDir, ndbcHistoricalTxtDir, ndbc45DayTxtDir,
// ndbcNcDir, ignoreStationsBefore, testMode); //M4700 ~2 hrs, was ~3 hrs //last done 2017-04-28
// ndbcNcDir, ignoreStationsBefore, testMode); //M4700 ~2 hrs, was ~3 hrs //last done 2017-05-25
test31201Nc(ndbcNcDir);
test41009Nc(ndbcNcDir);
test41015Nc(ndbcNcDir);
Expand All @@ -2818,7 +2818,7 @@ public static void main(String args[]) throws Exception {
//but 45 is more likely to get more information (if needed and if available)
//(45 days takes 25 minutes)
testMode = false; //always run 'true' then run 'false'
addLastNDaysInfo(ndbcNcDir, 5, testMode); //usually 5
//addLastNDaysInfo(ndbcNcDir, 5, testMode); //usually 5
//!!!!**** EACH MONTH, THIS TEST NEED UPDATING
test46088AddLastNDaysNc(ndbcNcDir);

Expand All @@ -2830,7 +2830,7 @@ public static void main(String args[]) throws Exception {
tar zxvf ndbcMett.tgz
as su
chown -R tomcat:erddap ndbcMett
chmod -R a+rw ndbcMett
chmod -R g+rw ndbcMett
rename ndbcMet ndbcMetR20150224 ndbcMet
rename ndbcMett ndbcMet ndbcMett
rm ndbcMett.tgz
Expand Down
Loading

0 comments on commit e34fa3b

Please sign in to comment.