Feature/bufr dump.v1.3.0 dm addwinds

docs/Release_Notes.txt

@@ -1,5 +1,141 @@
+bufr-dump v1.3.0        RELEASED        OCT 04,2024
+          v1.3.0        IMPLEMENTED     ??? ??,2024
+
+PRELUDE
+ This update (obsproc v1.3.0, bufr-dump v1.3.0, prepobs v1.2.0) 
+ introduces the following new dump files configuration:
+  sfcsno - snow from synoptic tanks messages containing snow info
+  snomad - snow from with MADIS msonet tank containing snow info
+  wndtwr - wind from US wind towers observation (dev tanks to be promo to prod)
+  sofarw - marine wave SOFAR observations
+ and splits 
+  satwnd - satwnd,satwn1,satwn2 to speed up processing
+
+FILE CHANGES
+ M docs/Release_Notes.txt
+ M fix/bufr_dumplist
+ A sorc/bufr_dupsno.fd (dupsno.f, CMakeLists.txt)
+ M sorc/CMakeLists.txt
+ M ush/dumpjb
+ M versions/VERSION
+ (M-modified; A-added; D-deleted)
+
+DOCS CHANGES
+ docs/Release_Notes.txt
+ Updated for this release.
+
+FIX CHANGES
+ fix/bufr_dumplist
+  Introduce new dump files configuration:
+  sfcsno - snow from synoptic tanks messages containing snow info
+  snomad - snow from with MADIS msonet tank containing snow info
+  wndtwr - wind from US wind towers observation
+  sofarw - marine wave SOFAR observations
+  Splits satwnd - satwnd,satwn1,satwn2 to speed up processing
+
+SOURCE CHANGES
+ sorc/bufr_dupsno.fd/dupsno.f
+ Introduce new duplicate check prioritizing snow-containing messages from the synop tanks
+
+ sorc/CMakeLists.txt
+ Add bufr_dumpsno.fd
+
+VERSIONS CHANGES
+ version/VERSION
+ Updated to this release's version
+
+CHANGES TO OUTPUT AND FILE SIZES
+ New dumps:
+ sfcsno(30MB), snomad(2MB), wndtwr (500kb), sofarw (500KB, often less) 
+
+PRE-IMPLEMENTATION TESTING REQUIREMENTS
+ obsproc v1.3.0 needs modules bufr-dump v1.3.0 and prepobs v1.2.0 in order to run properly
+ See docs/README.md for build instructions
+ See versions/build.ver and versions/run.ver for libaries
+
+ Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST
+ for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS and RRFS
+
+ Diference in file size should be seen only for the *msonet* and *gpsro* bufr dumps.
+
+DISSEMINATION INFORMATION
+ No changes to dissemination on WCOSS2
+
+SPECIAL INSTRUCTIONS
+ Make sure that the following binary fix files (from the old VLab repo) are downloaded:
+    nam_expdomain_halfdeg_imask.gbl
+    nam_expdomain_guam_halfdeg_imask.gbl
+    wave_landchxh
+ These files are currently hosted on the EMC FTP server at:
+ https://ftp.emc.ncep.noaa.gov/static_files/public/obsproc/bufr-dump-fix-HASH.tgz
+ They can be downloaded manually and into the cloned
+ repository by executing: ./ush/download_fix.sh
+
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+bufr-dump v1.2.1        RELEASED        JUN 26,2024
+          v1.2.1        IMPLEMENTED     SEP 05,2024
+
+PRELUDE
+ This update will continue to support obsproc.v1.2.0 and prepobs.v1.1.0 releases (it won't work with earlier obsproc and prepbufr versions!)
+
+ It augments the contents of two existing files:
+ *gpsro*  - relax collocation criteria to eliminate near-duplicates in the EUMETSAT SPIRE data
+ *msonet* - all South Dacota mesonet data (SD DOT)
+
+FILE CHANGES
+M docs/Release_Notes.txt
+M fix/bufr_dumplist
+M sorc/bufr_dupsat.fd/dupsat.f
+M versions/VERSION
+(M-modified; A-added; D-deleted)
+
+DOCS CHANGES
+ docs/Release_Notes.txt
+ Updated for this release.
+
+FIX CHANGES
+ fix/bufr_dumplist
+ _msonet: add data from tank b255/xx034
+
+SOURCE CHANGES
+ sorc/bufr_dupsat.fd/dupsat.f
+ Relax the collocation criteria for duplicates (DEXY,DSEC)
+
+VERSIONS CHANGES
+ version/VERSION
+ Updated to this release's version
+
+CHANGES TO OUTPUT AND FILE SIZES
+ Update dumps:
+ *msonet* - add subset NC255034, size is not impacted noticeably b/c of recent decoders updates which reduced the size of tanks 255/003,004,030 and *msonet* dumps.
+ *gpsro*  - add the EUMETSAT SPIRE data w.out duplicates
+
+PRE-IMPLEMENTATION TESTING REQUIREMENTS
+ obsproc v1.2.0 needs modules bufr-dump v1.2.* and prepobs v1.1.0 in order to run
+ See docs/README.md for build instructions
+ See versions/build.ver and versions/run.ver for libaries
+
+ Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST
+ for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS
+
+ Diference in file size should be seen only for the *msonet* and *gpsro* bufr dumps.
+
+DISSEMINATION INFORMATION
+ No changes to dissemination on WCOSS2
+
+SPECIAL INSTRUCTIONS
+ Make sure that the following binary fix files (from the old VLab repo) are downloaded:
+    nam_expdomain_halfdeg_imask.gbl
+    nam_expdomain_guam_halfdeg_imask.gbl
+    wave_landchxh
+ These files are currently hosted on the EMC FTP server at:
+ https://ftp.emc.ncep.noaa.gov/static_files/public/obsproc/bufr-dump-fix-HASH.tgz
+ They can be downloaded manually and into the cloned
+ repository by executing: ./ush/download_fix.sh
+
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 bufr-dump v1.2.0        RELEASED        AUG ??,2023
-          v1.2.?        IMPLEMENTED     ??? ??,2023
+          v1.2.0        IMPLEMENTED     MAY 22,2023
 
 PRELUDE
 

fix/bufr_dumplist

@@ -35,8 +35,8 @@
 
    --> Surface data - land -- TYPE 000
        -------------------------------
-_sflnd     nem 000000 000001 000002 000007 000008 000010 000011 000012 000015 000100 000101 000102
-_000       mgt 000000 000001 000002 000007 000008 000010 000011 000012 000015 000100 000101 000102
+_sflnd     nem 000000 000001 000002 000007 000008 000010 000011 000012 000015 000021 000100 000101 000102
+_000       mgt 000000 000001 000002 000007 000008 000010 000011 000012 000015 000021 000100 000101 000102
 
    --> Surface data - sea -- TYPE 001
        ------------------------------
@@ -45,8 +45,8 @@ _001       mgt 001001 001002 001003 001004 001005 001006 001007 001008 001009 00
 
    --> Vertical soundings (other than satellite) -- TYPE 002
        -----------------------------------------------------
-_vsndn     nem 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002101 002102 002103 002104 002105 002109
-_002       mgt 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002101 002102 002103 002104 002105 002109
+_vsndn     nem 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002020 002021 002101 002102 002103 002104 002105 002109
+_002       mgt 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002020 002021 002101 002102 002103 002104 002105 002109
 
    --> Vertical soundings (satellite) -- TYPE 003
        ------------------------------------------
@@ -55,8 +55,8 @@ _003       mgt 003001 003002 003003 003010 003101 003102 003104
 
    --> Single level upper-air data (other than satellite) -- TYPE 004
        --------------------------------------------------------------
-_sluan     nem 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004103
-_004       mgt 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004103
+_sluan     nem 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004017 004103
+_004       mgt 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004017 004103
 
    --> Single-level upper-air data (satellite) -- TYPE 005
        ---------------------------------------------------
@@ -115,7 +115,7 @@ _255       mgt 255001 255002 255003 255004 255005 255006 255007 255008 255009 25
          proper columnar convention.
 
 _aircar    nem 004004 004007
-_aircft    nem 004001 004002 004003 004006 004009 004010 004011 004103
+_aircft    nem 004001 004002 004003 004006 004009 004010 004011 004017 004103
 _adpsfc    nem 000000 000001 000002 000007 000100 000101 000102
 _adpupa    nem 002001 002002 002003 002004 002005 004005
 _uprair    nem 002101 002102 002103 002104 002105
@@ -130,12 +130,14 @@ _rassda    nem 002012
 _vadwnd    nem 002008 002017
 _goesnd    nem 003002 003003
 _goesfv    nem 003003
-_satwnd    nem 005021 005022 005023 005044 005045 005046 005067 005068 005069 005070 005071 005080 005081 005091
+_satwnd    nem 005044 005045 005046
+_satwn1    nem 005067 005068 005069
+_satwn2    nem 005070 005071 005080 005081 005091
 _sfcshp    nem 001001 001013 001002 001003 001004 001007 001102 001103 001101 001113 001104
 _sfcbog    nem 001006
 _gpsipw    nem 012004
-_msonet    nem 255001 255002 255003 255004 255005 255006 255007 255008 255009 255010 255011 255012 255014 255015 255016 255017 255018 255019 255020 255021 255022 255023 255024 255025 255026 255027 255028 255029 255031 255101
-_msone1    nem 255030 
+_msonet    nem 255001 255002 255003 255004 255005 255006 255007 255008 255009 255010 255011 255012 255014 255015 255016 255017 255018 255019 255020 255021 255022 255023 255024 255025 255026 255027 255028 255029 255031 255034 255101
+_msone1    nem 255030
 _coop      nem 255101 255102
 _hydro     nem 255131 255160 
 _radwnd    nem 006001 006002
@@ -150,7 +152,10 @@ _avcl18    nem 021054
 _shpall    nem 001001 001013
 _satmar    nem 031104 031106 031110 031112 031113 031114 031115 031117 031118 031119 031120 031121 031122 031123 031124 031125 031126 031127 031130 
 _satwhr    nem 005052 005053 005054 005055 005056 005099
-
+_wndtwr    nem 000021 002020 002021
+_sofarw    nem 001121
+_sfcsno    nem 000000 000001 000002 000007 000100 000101 000102
+_snomad    nem 255161
  ===================================================================================================================================
 
  Section 3. List of Individual Types and Subtypes
@@ -338,6 +343,7 @@ _tmdarp    nem 004012  #> TAMDAR-PenAir aircraft data (from GSD, netCDF)   | grn
 _tmdarc    nem 004013  #> TAMDAR-Chautauqua aircraft data(from GSD, netCDF)| grn 50   50 YYYY| grn 50   50 YYYY| grn 50   50 YYYY|     |     |
 _famdar    nem 004014  #> French AMDAR aircraft data (decoded from BUFR)   | grn 50   50 YYYY| grn 50   50 YYYY| grn 50   50 YYYY|     |     |
 _hdob      nem 004015  #> High Density OBs from reconnaissance acft data   | grn 50   50 YYYY| grn 50   50 YYYY| grn 50   50 YYYY|     |     |
+_affir     nem 004017  #> AFFIRS AMDAR acft data                           | grn 50   50 YYYY| grn 50   50 YYYY| grn 50   50 YYYY|     |     |
 _amdarb    nem 004103  #> AMDAR aircraft data (decoded from BUFR)          |     50   50 YYYY| YEL 50   50 YYYY| YEL 50   50 YYYY| YEL |     |
                                                                            |                 |                 |                 |     |     |
 _visuw     nem 005008  #> GOES/UW-CIMSS visible derived cloud motion       | grn 50   50 YYYY| grn 50   50 YYYY| grn 50   50 YYYY|     |     |

sorc/CMakeLists.txt

@@ -38,6 +38,7 @@ add_subdirectory(bufr_dupsat.fd)
 add_subdirectory(bufr_dupshp.fd)
 add_subdirectory(bufr_dupsst.fd)
 add_subdirectory(bufr_dupsyp.fd)
+add_subdirectory(bufr_dupsno.fd)
 add_subdirectory(bufr_duptac.fd)
 add_subdirectory(bufr_dupupr.fd)
 add_subdirectory(bufr_edtbfr.fd)

sorc/bufr_dupsat.fd/dupsat.f

@@ -451,7 +451,7 @@ PROGRAM BUFR_DUPSAT
       CALL W3TAGB('BUFR_DUPSAT',2022,0084,1100,'NP22')
 
       print *
-      print * ,'---> Welcome to BUFR_DUPSAT - Version 03-25-2022'
+      print * ,'---> Welcome to BUFR_DUPSAT - Version 06-07-2024'
       print *
 
       CALL DATELEN(10)
@@ -588,6 +588,16 @@ PROGRAM BUFR_DUPSAT
          IF(ATOVS.AND..NOT.DEXY_READ) DEXY = 0.01
       ENDIF
 
+
+C  RELAX GPS-RO DEXY and DSEC to "catch" and remove SPIRE EUMS and EKMI near-duplicates
+C      print *, 'SUBSET IS ', SUBSET
+      IF(SUBSET.EQ.'NC003010')  THEN
+         DEXY = 0.05
+         DSEC = 50
+C         print *, '--> BUFR_DUPSAT:  GPS RO is relaxed due to SPIRE :'
+C         print *, ' ILIANA DEXY=0.05, DSEC=50'
+      ENDIF
+
       IF(BDATE.NE.99999999.00_8) THEN
          PRINT 200, ADATE,BDATE
       ELSE

sorc/bufr_dupsno.fd/CMakeLists.txt

@@ -0,0 +1,12 @@
+list(APPEND fortran_src
+  dupsno.f
+)
+
+set(exe_name bufr_dupsno)
+add_executable(${exe_name} ${fortran_src})
+target_link_libraries(
+  ${exe_name}
+  w3emc::w3emc_4
+  bufr::bufr_4)
+
+install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})