Feature/b2i marine and testing

dump/mapping/bufr_marine_insitu_obs_builder.py

@@ -66,35 +66,101 @@ def get_station_basin(self, lat, lon):
         dlon = self.longitudes[1] - self.longitudes[0]
 
         # the data may be a masked array
-        ocean_basin = []
+        ocean_basin = ma.array([0]*lat.size, mask=lat.mask, dtype=np.int32)
         for i in range(n):
             if not ma.is_masked(lat[i]):
                 i1 = round((lat[i] - lat0) / dlat)
                 i2 = round((lon[i] - lon0) / dlon)
-                ocean_basin.append(self.basin_array[i1][i2])
-        return np.array(ocean_basin, dtype=np.int32)
+                ocean_basin[i] = self.basin_array[i1][i2]
+        return ocean_basin
+
+
+def clean_lat_lon(lat, lon):
+    """
+    return a mask for valid pairs of latitude and longitude.
+
+    Parameters:
+    lat : array-like or None
+        Latitude values, expected in [-90, 90].
+    lon : array-like or None
+        Longitude values, all in [-180, 180] or all in [0, 360].
+
+    Returns:
+    mask : numpy.ndarray
+        Boolean mask (same shape as inputs), True for valid lat/lon pairs.
+        Use to filter other arrays (e.g., other_data[mask]).
+    """
+    # Handle undefined or None inputs
+    if lat is None or lon is None:
+        return np.array([], dtype=bool)
+
+    # Convert inputs to NumPy arrays, preserving masked arrays
+    try:
+        lat = np.asarray(lat, dtype=float)
+        lon = np.asarray(lon, dtype=float)
+    except (ValueError, TypeError):
+        return np.zeros_like(lat, dtype=bool)
+
+    # Ensure arrays have the same shape
+    # probably needs to throw an exception
+    if lat.shape != lon.shape:
+        return np.zeros_like(lat, dtype=bool)
+
+    # Initialize mask (True for valid, False for invalid)
+    mask = np.ones(lat.shape, dtype=bool)
+
+    # Validate latitude: must be in [-90, 90]
+    mask &= (lat >= -90) & (lat <= 90) & ~np.isnan(lat)
+
+    # Validate longitude: all in [-180, 180] or all in [0, 360]
+    valid_lons = lon[mask]  # Longitudes where lat is valid
+    if len(valid_lons) == 0:
+        return mask
+
+    # Check longitude range consistency
+    in_neg180_180 = (valid_lons >= -180) & (valid_lons <= 180)
+    in_0_360 = (valid_lons >= 0) & (valid_lons <= 360)
+
+    # Determine if all valid longitudes are in one range
+    all_neg180_180 = np.all(in_neg180_180)
+    all_0_360 = np.all(in_0_360)
+
+    # If neither range is fully consistent, use dominant range
+    # if the fraction of the "other" range is too large,
+    # probably needs to throw an error
+    if not (all_neg180_180 or all_0_360):
+        if np.sum(in_neg180_180) >= np.sum(in_0_360):
+            mask &= (lon >= -180) & (lon <= 180)
+        else:
+            mask &= (lon >= 0) & (lon <= 360)
+    else:
+        if all_neg180_180:
+            mask &= (lon >= -180) & (lon <= 180)
+        else:
+            mask &= (lon >= 0) & (lon <= 360)
+
+    # Ensure no NaN in longitude
+    mask &= ~np.isnan(lon)
+    return mask
 
 
 class MarineInsituObsBuilder(ObsBuilder):
     def __init__(self, mapping_path, log_name=os.path.basename(__file__), config=None):
-        # print(f'===============================>>>><<<<')
-        # print(f'config = {config}')
-        # print(f'===============================>>>><<<<')
         self.ocean_basin_file = config['ocean_basin'] if 'ocean_basin' in config else None
-        # print(f'self.ocean_basin_file = {self.ocean_basin_file}')
-        # print(f'config = {config}')
-
         super().__init__(mapping_path, log_name=log_name, config=config)
 
     def make_obs(self, comm, input_path):
         container = super().make_obs(comm, input_path)
 
+        lat = container.get('latitude')
+        lon = container.get('longitude')
+        lat_lon_mask = clean_lat_lon(lat, lon)
+        container.apply_mask(lat_lon_mask)
+
         if self.ocean_basin_file and os.path.exists(self.ocean_basin_file):
             self._add_ocean_basin(container, self.ocean_basin_file)
         else:
             self.log.warning(f"No ocean basin file provided, or can not be found")
-
-        # print("MMMMMMMMMMMMMMMMMM")
         return container
 
     def _make_description(self):
@@ -109,6 +175,7 @@ def _make_description(self):
 
         return description
 
+
     def _add_preqc_var(self, container, name):
         v = container.get(name)
         paths = container.get_paths(name)
@@ -119,11 +186,7 @@ def _add_preqc_var(self, container, name):
     def _add_error_var(self, container, name, error):
         v = container.get(name)
         paths = container.get_paths(name)
-        # print(">>>>>>>>>>>>>>>>>>>>>>")
-        # print(f"_add_error_var {name}")
         error_var_name = f"ObsError{name}"
-        # print(f"_add_error_var {error_var_name}")
-        # print(">>>>>>>>>>>>>>>>>>>>>>")
         error_var = np.full_like(v, error)
         container.add(error_var_name, error_var, paths)
 
@@ -143,4 +206,3 @@ def _add_ocean_basin(self, container, nc_file_path):
         ocean = OceanBasin(nc_file_path)
         v = ocean.get_station_basin(lat, lon)
         container.add("oceanBasin", v, paths)
-

dump/mapping/bufr_marine_insitu_profile_argo.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_profile_bathy.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_profile_glider.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_profile_tesac.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_profile_tropical.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_profile_xbtctd.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_surface_altkob.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_surface_cstgd.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 

dump/mapping/bufr_marine_insitu_surface_dbuoyb_drifter.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os
+import numpy as np
+
+import bufr
+from bufr.obs_builder import add_main_functions
+from bufr_marine_insitu_obs_builder import MarineInsituObsBuilder, map_path
+
+MAPPING_PATH = map_path('bufr_marine_insitu_surface_dbuoyb_drifter.yaml')
+
+
+'''
+buoy types for drifters:
+------------------------
+00      Unspecified drifting buoy
+01      Standard Lagrangian drifter (Global Drifter Programme)
+02      Standard FGGE type drifting buoy (non-Lagrangian meteorological drifting buoy)
+03      Wind measuring FGGE type drifting buoy (non-Lagrangian meteorological drifting buoy)
+04      Ice drifter
+05      SVPG Standard Lagrangian drifter with GPS (BUFR)
+06      SVP-HR drifter with high-resolution temperature or thermistor string (BUFR)
+10      ALACE (Autonomous Lagrangian Circulation Explorer)
+11      MARVOR (MARine VORtical profiler)
+12      RAFOS (Ranging and Fixing of Sound)
+13      PROVOR (Profiling float with Argos)
+14      SOLO (Swimbladder-Operated Lagrangian Oscillating)
+15      APEX (Autonomous Profiling Explorer)
+'''
+
+drifter_buoy_types = [0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 13, 14, 15] 
+
+
+class MarineInsituSurfaceDrifterObsBuilder(MarineInsituObsBuilder):
+    def __init__(self, config=None):
+        super().__init__(MAPPING_PATH, config=config, log_name=os.path.basename(__file__))
+
+    def make_obs(self, comm, input_path):
+        # Get container from mapping file first
+        container = super().make_obs(comm, input_path)
+
+        temp = container.get("seaSurfaceTemperature")
+        temp_mask = (temp > -10.0) & (temp < 50.0)
+
+        buoy_type = container.get("buoyType")
+        rpid = container.get("stationID")
+
+        # rpid = stationID: string array (e.g., 'A8xxx')
+        # buoy_type: int array (e.g., 1, 2, 3), etc.
+        drifter_mask = np.isin(buoy_type, drifter_buoy_types, assume_unique=True)
+
+        # Optional: Add RPID check for drifter patterns (e.g., starts with 'A8')
+        rpid_drifter_mask = np.array([isinstance(r, str) and r.startswith('A8') for r in rpid.filled('')])
+        drifter_mask = drifter_mask | rpid_drifter_mask
+
+        # Handle masked (missing) BUYT values
+        # If BUYT is masked, assume not a drifter unless RPID suggests otherwise
+        drifter_mask = np.where(buoy_type.mask, rpid_drifter_mask, drifter_mask)
+
+        # print("BBBBBBBBBBBBBBBBBBB")
+        # print(buoy_mask)
+        # print(temp_mask.size)
+        # print(buoy_type.size)
+        # print(buoy_mask.size)
+        # print(np.count_nonzero(buoy_mask))
+        # print(np.count_nonzero(temp_mask))
+        # print("BBBBBBBBBBBBBBBBBBB")
+        # container.apply_mask(buoy_mask & temp_mask)
+
+        container.apply_mask(drifter_mask & temp_mask)
+
+        self._add_preqc_var(container, "seaSurfaceTemperature")
+        self._add_error_var(container, "seaSurfaceTemperature", error=0.24)
+
+        return container
+
+add_main_functions(MarineInsituSurfaceDrifterObsBuilder)

dump/mapping/bufr_marine_insitu_surface_dbuoyb_drifter.yaml

@@ -0,0 +1,111 @@
+bufr:
+  variables:
+    dateTime:
+      datetime:
+        year: "*/YEAR"
+        month: "*/MNTH"
+        day: "*/DAYS"
+        hour: "*/HOUR"
+        minute: "*/MINU"
+
+    receiptTime:
+      datetime:
+        year: "[*/RCYR, */RCPTIM{1}/RCYR]"
+        month: "[*/RCMO, */RCPTIM{1}/RCMO]"
+        day: "[*/RCDY, */RCPTIM{1}/RCDY]"
+        hour: "[*/RCHR, */RCPTIM{1}/RCHR]"
+        minute: "[*/RCMI, */RCPTIM{1}/RCMI]"
+
+    stationID:
+        query: "*/RPID"
+
+    buoyType:
+        query: "*/BUYT"
+
+    latitude:
+        query: "*/CLATH"
+
+    longitude:
+        query: "*/CLONH"
+
+    seaSurfaceTemperature:
+        query: "*/BBYSSTS/SST0"
+        transforms:
+          - offset: -273.15   # convert from Kelvin to Celsius
+
+
+encoder:
+  globals:
+    - name: "source"
+      type: string
+      value: "NCEP data tank"
+
+    - name: "description"
+      type: string
+      value: "6-hrly in situ drifters surface"
+
+    - name: "platformLongDescription"
+      type: string
+      value: "Drifters surface obs from dbuoyb: temperature"
+
+    - name: "data_providerOrigin"
+      type: string
+      value: "U.S. NOAA"
+
+    - name: "Converter"
+      type: string
+      value: "BUFR to IODA Converter"
+
+    - name: "_ioda_layout_version"
+      type: int
+      value: 0
+
+    - name: "_ioda_layout"
+      type: string
+      value: "ObsGroup"
+
+  variables:
+    - name: "MetaData/dateTime"
+      source: variables/dateTime
+      longName: "dateTime"
+      units: "seconds since 1970-01-01T00:00:00Z"
+
+    - name: "MetaData/rcptdateTime"
+      source: variables/receiptTime
+      longName: "receipt Datetime"
+      units: "seconds since 1970-01-01T00:00:00Z"
+
+    - name: "MetaData/stationID"
+      source: variables/stationID
+      longName: "Station Identification"
+      units: ""
+
+    - name: "MetaData/BuoyType"
+      source: variables/buoyType
+      longName: "Buoy Type"
+      units: ""
+
+    - name: "MetaData/latitude"
+      source: variables/latitude
+      longName: "Latitude"
+      units: "degrees_north"
+
+    - name: "MetaData/longitude"
+      source: variables/longitude
+      longName: "Longitude"
+      units: "degrees_east"
+
+    - name: "ObsValue/seaSurfaceTemperature"
+      source: variables/seaSurfaceTemperature
+      longName: "Sea Surface Temperature"
+      units: "degC"
+
+    - name: "ObsError/seaSurfaceTemperature"
+      source: variables/ObsErrorseaSurfaceTemperature
+      longName: "Sea Surface Temperature Error"
+      units: "degC"
+
+    - name: "PreQC/seaSurfaceTemperature"
+      source: variables/PreQCseaSurfaceTemperature
+      longName: "PreQC Sea Surface temperature"
+      units: ""

dump/mapping/bufr_marine_insitu_surface_drifter.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np
 
@@ -36,7 +38,7 @@ def make_obs(self, comm, input_path):
         print(np.count_nonzero(buoy_mask))
         print(np.count_nonzero(temp_mask))
         print("BBBBBBBBBBBBBBBBBBB")
-        # container.apply_mask(buoy_mask & temp_mask)
+        container.apply_mask(buoy_mask & temp_mask)
 
         self._add_preqc_var(container, "seaSurfaceTemperature")
         self._add_preqc_var(container, "salinity")

dump/mapping/bufr_marine_insitu_surface_lcman.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import numpy as np