Merge pull request #226 from ChrisJohnNOAA/parquet_fill_value

Have the parquet export transform from fill values to mising values
ERDDAP · Nov 7, 2024 · 1ef0d53 · 1ef0d53
2 parents 44c825e + ae20e14
commit 1ef0d53
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 9 deletions.
diff --git a/WEB-INF/classes/com/cohort/array/PAOne.java b/WEB-INF/classes/com/cohort/array/PAOne.java
@@ -39,20 +39,23 @@ public PAOne(PAType paType, String value) {
   /** This constructs a paOne (with a value of 0) of the same type as the PrimitiveArray. */
   public PAOne(PrimitiveArray otherPA) {
     pa = PrimitiveArray.factory(otherPA.elementType(), 1, true);
+    pa.setMaxIsMV(otherPA.getMaxIsMV());
     elementSize = pa.elementSize();
   }
 
   /** This constructs a paOne with the specified value. */
   public PAOne(PrimitiveArray otherPA, int index) {
     pa = PrimitiveArray.factory(otherPA.elementType(), 1, true);
     pa.setFromPA(0, otherPA, index);
+    pa.setMaxIsMV(otherPA.getMaxIsMV());
     elementSize = pa.elementSize();
   }
 
   /** This makes a new PAOne which is a copy of the other PAOne. */
   public PAOne(PAOne tPAOne) {
     pa = PrimitiveArray.factory(tPAOne.pa.elementType(), 1, true);
     pa.setFromPA(0, tPAOne.pa, 0);
+    pa.setMaxIsMV(tPAOne.pa.getMaxIsMV());
     elementSize = pa.elementSize();
   }
 

diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java
@@ -16167,14 +16167,11 @@ private void addMetadata(Map<String, String> metadata, Attributes attributes, St
         continue;
       }
       PrimitiveArray tValue = attributes.get(tName);
-      if (tValue == null || tValue.size() == 0 || tValue.toString().length() == 0) {
-        continue; // do nothing
-      }
       if ("time_".equalsIgnoreCase(prefix)
           && Calendar2.SECONDS_SINCE_1970.equals(attributes.getString(tName))) {
         metadata.put(prefix + tName, Calendar2.MILLISECONDS_SINCE_1970);
       } else {
-        metadata.put(prefix + tName, tValue.toCSVString());
+        metadata.put(prefix + tName, tValue == null ? null : tValue.toString());
       }
     }
   }
@@ -16188,6 +16185,7 @@ private void addMetadata(Map<String, String> metadata, Attributes attributes, St
   public void writeParquet(String fullFileName, boolean fullMetadata) throws Exception {
     String msg = "  Table.writeParquet " + fullFileName;
     long time = System.currentTimeMillis();
+    convertToStandardMissingValues();
 
     int randomInt = Math2.random(Integer.MAX_VALUE);
     MessageType schema = getParquetSchemaForTable();

diff --git a/src/test/java/gov/noaa/pfel/coastwatch/pointdata/TableTests.java b/src/test/java/gov/noaa/pfel/coastwatch/pointdata/TableTests.java
@@ -20913,9 +20913,9 @@ void testWriteParquet() throws Exception {
         "aString,aChar,aByte,aUByte,aShort,aUShort,anInt,aUInt,aLong,aULong,aFloat,aDouble\n"
             + "a\\u00fcb\\nc\\td\\u20ace,\\u00fc,-128,0,-32768,0,-2147483648,0,-9223372036854775808,0.0,-3.4028235E38,-1.7976931348623157E308\n"
             + "ab,\\u0000,0,127,0,32767,0,7,0,1.0,2.2,3.3\n"
-            + ",A,99,99,9999,9999,999999999,2147483647,8,9.223372036854776E18,1.4E-45,4.9E-324\n"
+            + ",A,,,,,,2147483647,8,9.223372036854776E18,1.4E-45,4.9E-324\n"
             + "cd,\\t,126,254,32766,65534,2147483646,4294967294,9223372036854775806,1.8446744073709552E19,3.4028235E38,1.7976931348623157E308\n"
-            + ",\\u20ac,,,,,,4294967295,,,,\n";
+            + ",\\u20ac,,,,,,,,,,\n";
     Test.ensureEqual(results, expected, "results=\n" + results);
     PAType tTypes[] = {
       PAType.STRING, PAType.STRING, PAType.BYTE, PAType.SHORT,
@@ -20954,9 +20954,9 @@ void testWriteParquetWithMetadata() throws Exception {
         "aString,aChar,aByte,aUByte,aShort,aUShort,anInt,aUInt,aLong,aULong,aFloat,aDouble\n"
             + "a\\u00fcb\\nc\\td\\u20ace,\\u00fc,-128,0,-32768,0,-2147483648,0,-9223372036854775808,0.0,-3.4028235E38,-1.7976931348623157E308\n"
             + "ab,\\u0000,0,127,0,32767,0,7,0,1.0,2.2,3.3\n"
-            + ",A,99,99,9999,9999,999999999,2147483647,8,9.223372036854776E18,1.4E-45,4.9E-324\n"
+            + ",A,,,,,,2147483647,8,9.223372036854776E18,1.4E-45,4.9E-324\n"
             + "cd,\\t,126,254,32766,65534,2147483646,4294967294,9223372036854775806,1.8446744073709552E19,3.4028235E38,1.7976931348623157E308\n"
-            + ",\\u20ac,,,,,,4294967295,,,,\n";
+            + ",\\u20ac,,,,,,,,,,\n";
     Test.ensureEqual(results, expected, "results=\n" + results);
     PAType tTypes[] = {
       PAType.STRING, PAType.STRING, PAType.BYTE, PAType.SHORT,

diff --git a/src/test/java/jetty/JettyTests.java b/src/test/java/jetty/JettyTests.java
@@ -12693,7 +12693,11 @@ void testOpendap() throws Throwable {
               + //
               "  :geospatial_lon_units = \"degrees_east\";\n"
               + //
-              "  :history = \"Tue Apr 30 05:42:52 HST 2024 : imported by GrADS Data Server 2.0\n";
+              "  :history = \"DDD MMM dd hh:mm:ss ZZZ YYYY : imported by GrADS Data Server 2.0\n";
+      results =
+          results.replaceAll(
+              "\\w{3} \\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2} \\w{3} \\d{4}",
+              "DDD MMM dd hh:mm:ss ZZZ YYYY");
       // int po = results.indexOf(":history = \"NASA GSFC (OBPG)\n");
       // Test.ensureTrue(po > 0, "RESULTS=\n" + results);
       // Test.ensureLinesMatch(results.substring(0, po + 29), expected, "RESULTS=\n" +