YAMLs and Python script for the AVHRR BUFR

dump/mapping/bufr_avhrr.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import os
+import numpy as np
+
+import bufr
+from bufr.obs_builder import ObsBuilder, add_main_functions, map_path
+
+MAPPING_PATH = map_path('bufr_avhrr.yaml')
+
+
+class BufrGsrasrObsBuilder(ObsBuilder):
+
+    def __init__(self):
+        super().__init__(MAPPING_PATH, log_name=os.path.basename(__file__))
+
+    def make_obs(self, comm, input_path):
+        # Get container from mapping file
+        self.log.info('Get container from bufr')
+        container = super().make_obs(comm, input_path)
+
+        self.log.debug(f'Container list (original): {container.list()}')
+        self.log.debug(f'All_sub_categories =  {container.all_sub_categories()}')
+        self.log.debug(f'Category map = {container.get_category_map()}')
+
+        for cat in container.all_sub_categories():
+            self.log.debug(f'category: {cat}')
+
+            satId = container.get('satelliteId', cat)
+
+            if not np.any(satId):
+                self.log.warning(f'Category {cat[0]} does not exist in input file')
+                continue  # Skip invalid category
+
+        # Final container state
+        self.log.debug(f'Container list (updated): {container.list()}')
+
+        return container
+
+    def _make_description(self):
+        description = super()._make_description()
+        return description
+
+
+add_main_functions(BufrGsrasrObsBuilder)

dump/mapping/bufr_avhrr.yaml

@@ -0,0 +1,156 @@
+bufr:
+  subsets:
+    - NC021051
+    - NC021053
+
+  variables:
+    # MetaData
+    timestamp:
+      datetime:
+        year: "*/YEAR"
+        month: "*/MNTH"
+        day: "*/DAYS"
+        hour: "*/HOUR"
+        minute: "*/MINU"
+        second: "*/SECO"
+
+    latitude:
+      query: "*/CLATH"
+
+    longitude:
+      query: "*/CLONH"
+
+    satelliteId:
+      query: "*/SAID"
+
+    satelliteZenithAngle:
+      query: "*/SAZA"
+
+    solarZenithAngle:
+      query: "*/SOZA"
+
+    fieldOfViewNumber:
+      query: "*/FOVN"
+
+    cloudFromAVHRRCloudMask:
+      query: "*/CLAVR"
+
+    sensorChannelNumber:
+      query: "*/AVCSEQ/INCN"
+
+    albedo:
+      query: "*/AVCSEQ/ALBD"
+
+    # ObsValue
+    brightnessTemperature:
+      query: "*/AVCSEQ/TMBR"
+
+
+  splits:
+    satId:
+      category:
+        variable: satelliteId
+        map:
+          _3: metop-b
+          _4: metop-a
+          _5: metop-c
+
+encoder:
+  globals:
+    - name: "description"
+      type: string
+      value: ""
+
+    - name: "platformCommonName"
+      type: string
+      value: "EUMETSAT MetOp and NESDIS NOAA series"
+
+    - name: "platformLongDescription"
+      type: string
+      value: "IR Imager Cross-Scanning Polar-Orbiting - MetOp and NOAA series"
+
+    - name: "sensorCommonName"
+      type: string
+      value: "AVHRR"
+
+    - name: "sensorLongDescription"
+      type: string
+      value: ""
+
+    - name: "source"
+      type: string
+      value: "U.S. National Weather Service, National Centres for Environmental Prediction (NCEP))"
+
+    - name: "sourceFiles"
+      type: string
+      value: "NCEP BUFR Dump - NC021051 (NOAA, MetOp); NC021051 (NOAA), NC005081 (MetOp)"
+
+    - name: "processingLevel"
+      type: string
+      value: "Level-2"
+
+    - name: "converter"
+      type: string
+      value: "bufr-query"
+
+  variables:
+
+    # MetaData 
+    - name: "MetaData/dateTime"
+      source: variables/timestamp
+      longName: "Datetime"
+      units: "seconds since 1970-01-01T00:00:00Z"
+
+    - name: "MetaData/latitude"
+      source: variables/latitude
+      longName: "Latitude"
+      units: "degrees_north"
+      range: [-90, 90]
+
+    - name: "MetaData/longitude"
+      source: variables/longitude
+      longName: "Longitude"
+      units: "degrees_east"
+      range: [-180, 180]
+
+    - name: "MetaData/satelliteIdentifier"
+      source: variables/satelliteId
+      longName: "Satellite Identifier"
+
+    - name: "MetaData/satelliteZenithAngle"
+      source: variables/satelliteZenithAngle
+      longName: "Satellite Zenith Angle"
+      units: "degree"
+      range: [0, 90]
+
+    - name: "MetaData/solarZenithAngle"
+      source: variables/solarZenithAngle
+      longName: "Solar Zenith Angle"
+      units: "degree"
+      range: [0, 180]
+
+    - name: "MetaData/fieldOfViewNumber"
+      source: variables/fieldOfViewNumber
+      longName: "Field Of View Number"
+      units: ""
+
+    - name: "MetaData/cloudFromAVHRRCloudMask"
+      source: variables/cloudFromAVHRRCloudMask
+      longName: "Cloud From AVHRR Cloud Mask"
+      units: ""
+
+    - name: "MetaData/sensorChannelNumber"
+      source: variables/sensorChannelNumber
+      longName: "sensor Channel Number"
+      units: ""
+
+    - name: "MetaData/albedo"
+      source: variables/albedo
+      longName: "Albedo"
+      units: "%"
+
+    # ObsValue
+    - name: "ObsValue/brightnessTemperature"
+      source: variables/brightnessTemperature
+      longName: "Brightness Temperature (Hight Accurcay)"
+      units: "K"

ush/test/config/bufr_bufr4backend_avhrr.yaml

@@ -0,0 +1,61 @@
+time window:
+  begin: "2024-02-18T21:00:00Z"
+  end: "2024-02-19T03:00:00Z"
+  bound to include: begin
+
+observations:
+- obs space:
+    name: "avhrr-metop-b"
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: bufr
+        obsfile: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+        mapping file: "./bufr_avhrr.yaml"
+        category: ["metop-b"]
+        cache categories:            # optional
+          - ["metop-a"]
+          - ["metop-b"]
+          - ["metop-c"]
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/bufr4backend/gdas.t00z.avhrr_metop-b.tm00.nc"
+
+
+- obs space:
+    name: "avhrr-metop-c"
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: bufr
+        obsfile: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+        mapping file: "./bufr_avhrr.yaml"
+        category: ["metop-c"]
+        cache categories:            # optional
+          - ["metop-a"]
+          - ["metop-b"]
+          - ["metop-c"]
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/bufr4backend/gdas.t00z.avhrr_metop-c.tm00.nc"
+
+
+- obs space:
+    name: "avhrr-metop-a"
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: bufr
+        obsfile: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+        mapping file: "./bufr_avhrr.yaml"
+        category: ["metop-a"]
+        cache categories:            # optional
+          - ["metop-a"]
+          - ["metop-b"]
+          - ["metop-c"]
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/bufr4backend/gdas.t00z.avhrr_metop-a.tm00.nc"

ush/test/config/bufr_script4backend_avhrr.yaml

@@ -0,0 +1,62 @@
+time window:
+  begin: "2024-02-18T21:00:00Z"
+  end: "2024-02-19T03:00:00Z"
+  bound to include: begin
+
+observations:
+- obs space:
+    name: "avhrr-metop-b"
+    observed variables: [brightnessTemperature]
+    derived variables: [brightnessTemperature]
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: script
+        script file: "bufr_avhrr.py"
+        args:
+          input_path: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+          mapping_path: "./bufr_avhrr.yaml"
+            category: "metop-b"
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/script4backend/gdas.t00z.avhrr_metop-b.tm00.nc"
+
+
+- obs space:
+    name: "avhrr-metop-c"
+    observed variables: [brightnessTemperature]
+    derived variables: [brightnessTemperature]
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: script
+        script file: "bufr_avhrr.py"
+        args:
+          input_path: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+          mapping_path: "./bufr_avhrr.yaml"
+            category: "metop-c"
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/script4backend/gdas.t00z.avhrr_metop-c.tm00.nc"
+
+
+- obs space:
+    name: "avhrr-metop-a"
+    observed variables: [brightnessTemperature]
+    derived variables: [brightnessTemperature]
+    simulated variables: [brightnessTemperature]
+    obsdatain:
+      engine:
+        type: script
+        script file: "bufr_avhrr.py"
+        args:
+          input_path: "./testinput/2024021900/gdas.t00z.avcsam.tm00.bufr_d"
+          mapping_path: "./bufr_avhrr.yaml"
+            category: "metop-a"
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2024021900/script4backend/gdas.t00z.avhrr_metop-a.tm00.nc"
+