diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index fbc0382..d0cd07a 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -1,5 +1,141 @@ +bufr-dump v1.3.0 RELEASED OCT 04,2024 + v1.3.0 IMPLEMENTED ??? ??,2024 + +PRELUDE + This update (obsproc v1.3.0, bufr-dump v1.3.0, prepobs v1.2.0) + introduces the following new dump files configuration: + sfcsno - snow from synoptic tanks messages containing snow info + snomad - snow from with MADIS msonet tank containing snow info + wndtwr - wind from US wind towers observation (dev tanks to be promo to prod) + sofarw - marine wave SOFAR observations + and splits + satwnd - satwnd,satwn1,satwn2 to speed up processing + +FILE CHANGES + M docs/Release_Notes.txt + M fix/bufr_dumplist + A sorc/bufr_dupsno.fd (dupsno.f, CMakeLists.txt) + M sorc/CMakeLists.txt + M ush/dumpjb + M versions/VERSION + (M-modified; A-added; D-deleted) + +DOCS CHANGES + docs/Release_Notes.txt + Updated for this release. + +FIX CHANGES + fix/bufr_dumplist + Introduce new dump files configuration: + sfcsno - snow from synoptic tanks messages containing snow info + snomad - snow from with MADIS msonet tank containing snow info + wndtwr - wind from US wind towers observation + sofarw - marine wave SOFAR observations + Splits satwnd - satwnd,satwn1,satwn2 to speed up processing + +SOURCE CHANGES + sorc/bufr_dupsno.fd/dupsno.f + Introduce new duplicate check prioritizing snow-containing messages from the synop tanks + + sorc/CMakeLists.txt + Add bufr_dumpsno.fd + +VERSIONS CHANGES + version/VERSION + Updated to this release's version + +CHANGES TO OUTPUT AND FILE SIZES + New dumps: + sfcsno(30MB), snomad(2MB), wndtwr (500kb), sofarw (500KB, often less) + +PRE-IMPLEMENTATION TESTING REQUIREMENTS + obsproc v1.3.0 needs modules bufr-dump v1.3.0 and prepobs v1.2.0 in order to run properly + See docs/README.md for build instructions + See versions/build.ver and versions/run.ver for libaries + + Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST + for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS and RRFS + + Diference in file size should be seen only for the *msonet* and *gpsro* bufr dumps. + +DISSEMINATION INFORMATION + No changes to dissemination on WCOSS2 + +SPECIAL INSTRUCTIONS + Make sure that the following binary fix files (from the old VLab repo) are downloaded: + nam_expdomain_halfdeg_imask.gbl + nam_expdomain_guam_halfdeg_imask.gbl + wave_landchxh + These files are currently hosted on the EMC FTP server at: + https://ftp.emc.ncep.noaa.gov/static_files/public/obsproc/bufr-dump-fix-HASH.tgz + They can be downloaded manually and into the cloned + repository by executing: ./ush/download_fix.sh + +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +bufr-dump v1.2.1 RELEASED JUN 26,2024 + v1.2.1 IMPLEMENTED SEP 05,2024 + +PRELUDE + This update will continue to support obsproc.v1.2.0 and prepobs.v1.1.0 releases (it won't work with earlier obsproc and prepbufr versions!) + + It augments the contents of two existing files: + *gpsro* - relax collocation criteria to eliminate near-duplicates in the EUMETSAT SPIRE data + *msonet* - all South Dacota mesonet data (SD DOT) + +FILE CHANGES +M docs/Release_Notes.txt +M fix/bufr_dumplist +M sorc/bufr_dupsat.fd/dupsat.f +M versions/VERSION +(M-modified; A-added; D-deleted) + +DOCS CHANGES + docs/Release_Notes.txt + Updated for this release. + +FIX CHANGES + fix/bufr_dumplist + _msonet: add data from tank b255/xx034 + +SOURCE CHANGES + sorc/bufr_dupsat.fd/dupsat.f + Relax the collocation criteria for duplicates (DEXY,DSEC) + +VERSIONS CHANGES + version/VERSION + Updated to this release's version + +CHANGES TO OUTPUT AND FILE SIZES + Update dumps: + *msonet* - add subset NC255034, size is not impacted noticeably b/c of recent decoders updates which reduced the size of tanks 255/003,004,030 and *msonet* dumps. + *gpsro* - add the EUMETSAT SPIRE data w.out duplicates + +PRE-IMPLEMENTATION TESTING REQUIREMENTS + obsproc v1.2.0 needs modules bufr-dump v1.2.* and prepobs v1.1.0 in order to run + See docs/README.md for build instructions + See versions/build.ver and versions/run.ver for libaries + + Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST + for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS + + Diference in file size should be seen only for the *msonet* and *gpsro* bufr dumps. + +DISSEMINATION INFORMATION + No changes to dissemination on WCOSS2 + +SPECIAL INSTRUCTIONS + Make sure that the following binary fix files (from the old VLab repo) are downloaded: + nam_expdomain_halfdeg_imask.gbl + nam_expdomain_guam_halfdeg_imask.gbl + wave_landchxh + These files are currently hosted on the EMC FTP server at: + https://ftp.emc.ncep.noaa.gov/static_files/public/obsproc/bufr-dump-fix-HASH.tgz + They can be downloaded manually and into the cloned + repository by executing: ./ush/download_fix.sh + +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX bufr-dump v1.2.0 RELEASED AUG ??,2023 - v1.2.? IMPLEMENTED ??? ??,2023 + v1.2.0 IMPLEMENTED MAY 22,2023 PRELUDE diff --git a/fix/bufr_dumplist b/fix/bufr_dumplist index 6a95fce..ed576ab 100644 --- a/fix/bufr_dumplist +++ b/fix/bufr_dumplist @@ -35,8 +35,8 @@ --> Surface data - land -- TYPE 000 ------------------------------- -_sflnd nem 000000 000001 000002 000007 000008 000010 000011 000012 000015 000100 000101 000102 -_000 mgt 000000 000001 000002 000007 000008 000010 000011 000012 000015 000100 000101 000102 +_sflnd nem 000000 000001 000002 000007 000008 000010 000011 000012 000015 000021 000100 000101 000102 +_000 mgt 000000 000001 000002 000007 000008 000010 000011 000012 000015 000021 000100 000101 000102 --> Surface data - sea -- TYPE 001 ------------------------------ @@ -45,8 +45,8 @@ _001 mgt 001001 001002 001003 001004 001005 001006 001007 001008 001009 00 --> Vertical soundings (other than satellite) -- TYPE 002 ----------------------------------------------------- -_vsndn nem 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002101 002102 002103 002104 002105 002109 -_002 mgt 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002101 002102 002103 002104 002105 002109 +_vsndn nem 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002020 002021 002101 002102 002103 002104 002105 002109 +_002 mgt 002001 002002 002003 002004 002005 002006 002007 002008 002009 002010 002011 002012 002013 002014 002015 002016 002017 002020 002021 002101 002102 002103 002104 002105 002109 --> Vertical soundings (satellite) -- TYPE 003 ------------------------------------------ @@ -55,8 +55,8 @@ _003 mgt 003001 003002 003003 003010 003101 003102 003104 --> Single level upper-air data (other than satellite) -- TYPE 004 -------------------------------------------------------------- -_sluan nem 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004103 -_004 mgt 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004103 +_sluan nem 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004017 004103 +_004 mgt 004001 004002 004003 004004 004005 004006 004007 004008 004009 004010 004011 004012 004013 004014 004015 004017 004103 --> Single-level upper-air data (satellite) -- TYPE 005 --------------------------------------------------- @@ -115,7 +115,7 @@ _255 mgt 255001 255002 255003 255004 255005 255006 255007 255008 255009 25 proper columnar convention. _aircar nem 004004 004007 -_aircft nem 004001 004002 004003 004006 004009 004010 004011 004103 +_aircft nem 004001 004002 004003 004006 004009 004010 004011 004017 004103 _adpsfc nem 000000 000001 000002 000007 000100 000101 000102 _adpupa nem 002001 002002 002003 002004 002005 004005 _uprair nem 002101 002102 002103 002104 002105 @@ -130,12 +130,14 @@ _rassda nem 002012 _vadwnd nem 002008 002017 _goesnd nem 003002 003003 _goesfv nem 003003 -_satwnd nem 005021 005022 005023 005044 005045 005046 005067 005068 005069 005070 005071 005080 005081 005091 +_satwnd nem 005044 005045 005046 +_satwn1 nem 005067 005068 005069 +_satwn2 nem 005070 005071 005080 005081 005091 _sfcshp nem 001001 001013 001002 001003 001004 001007 001102 001103 001101 001113 001104 _sfcbog nem 001006 _gpsipw nem 012004 -_msonet nem 255001 255002 255003 255004 255005 255006 255007 255008 255009 255010 255011 255012 255014 255015 255016 255017 255018 255019 255020 255021 255022 255023 255024 255025 255026 255027 255028 255029 255031 255101 -_msone1 nem 255030 +_msonet nem 255001 255002 255003 255004 255005 255006 255007 255008 255009 255010 255011 255012 255014 255015 255016 255017 255018 255019 255020 255021 255022 255023 255024 255025 255026 255027 255028 255029 255031 255034 255101 +_msone1 nem 255030 _coop nem 255101 255102 _hydro nem 255131 255160 _radwnd nem 006001 006002 @@ -150,7 +152,10 @@ _avcl18 nem 021054 _shpall nem 001001 001013 _satmar nem 031104 031106 031110 031112 031113 031114 031115 031117 031118 031119 031120 031121 031122 031123 031124 031125 031126 031127 031130 _satwhr nem 005052 005053 005054 005055 005056 005099 - +_wndtwr nem 000021 002020 002021 +_sofarw nem 001121 +_sfcsno nem 000000 000001 000002 000007 000100 000101 000102 +_snomad nem 255161 =================================================================================================================================== Section 3. List of Individual Types and Subtypes @@ -338,6 +343,7 @@ _tmdarp nem 004012 #> TAMDAR-PenAir aircraft data (from GSD, netCDF) | grn _tmdarc nem 004013 #> TAMDAR-Chautauqua aircraft data(from GSD, netCDF)| grn 50 50 YYYY| grn 50 50 YYYY| grn 50 50 YYYY| | | _famdar nem 004014 #> French AMDAR aircraft data (decoded from BUFR) | grn 50 50 YYYY| grn 50 50 YYYY| grn 50 50 YYYY| | | _hdob nem 004015 #> High Density OBs from reconnaissance acft data | grn 50 50 YYYY| grn 50 50 YYYY| grn 50 50 YYYY| | | +_affir nem 004017 #> AFFIRS AMDAR acft data | grn 50 50 YYYY| grn 50 50 YYYY| grn 50 50 YYYY| | | _amdarb nem 004103 #> AMDAR aircraft data (decoded from BUFR) | 50 50 YYYY| YEL 50 50 YYYY| YEL 50 50 YYYY| YEL | | | | | | | | _visuw nem 005008 #> GOES/UW-CIMSS visible derived cloud motion | grn 50 50 YYYY| grn 50 50 YYYY| grn 50 50 YYYY| | | diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt index 8343272..bf6d7b8 100644 --- a/sorc/CMakeLists.txt +++ b/sorc/CMakeLists.txt @@ -38,6 +38,7 @@ add_subdirectory(bufr_dupsat.fd) add_subdirectory(bufr_dupshp.fd) add_subdirectory(bufr_dupsst.fd) add_subdirectory(bufr_dupsyp.fd) +add_subdirectory(bufr_dupsno.fd) add_subdirectory(bufr_duptac.fd) add_subdirectory(bufr_dupupr.fd) add_subdirectory(bufr_edtbfr.fd) diff --git a/sorc/bufr_dupsat.fd/dupsat.f b/sorc/bufr_dupsat.fd/dupsat.f index 5f0cb9a..4323287 100644 --- a/sorc/bufr_dupsat.fd/dupsat.f +++ b/sorc/bufr_dupsat.fd/dupsat.f @@ -451,7 +451,7 @@ PROGRAM BUFR_DUPSAT CALL W3TAGB('BUFR_DUPSAT',2022,0084,1100,'NP22') print * - print * ,'---> Welcome to BUFR_DUPSAT - Version 03-25-2022' + print * ,'---> Welcome to BUFR_DUPSAT - Version 06-07-2024' print * CALL DATELEN(10) @@ -588,6 +588,16 @@ PROGRAM BUFR_DUPSAT IF(ATOVS.AND..NOT.DEXY_READ) DEXY = 0.01 ENDIF + +C RELAX GPS-RO DEXY and DSEC to "catch" and remove SPIRE EUMS and EKMI near-duplicates +C print *, 'SUBSET IS ', SUBSET + IF(SUBSET.EQ.'NC003010') THEN + DEXY = 0.05 + DSEC = 50 +C print *, '--> BUFR_DUPSAT: GPS RO is relaxed due to SPIRE :' +C print *, ' ILIANA DEXY=0.05, DSEC=50' + ENDIF + IF(BDATE.NE.99999999.00_8) THEN PRINT 200, ADATE,BDATE ELSE diff --git a/sorc/bufr_dupsno.fd/CMakeLists.txt b/sorc/bufr_dupsno.fd/CMakeLists.txt new file mode 100644 index 0000000..87a44d3 --- /dev/null +++ b/sorc/bufr_dupsno.fd/CMakeLists.txt @@ -0,0 +1,12 @@ +list(APPEND fortran_src + dupsno.f +) + +set(exe_name bufr_dupsno) +add_executable(${exe_name} ${fortran_src}) +target_link_libraries( + ${exe_name} + w3emc::w3emc_4 + bufr::bufr_4) + +install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/sorc/bufr_dupsno.fd/dupsno.f b/sorc/bufr_dupsno.fd/dupsno.f new file mode 100755 index 0000000..3730439 --- /dev/null +++ b/sorc/bufr_dupsno.fd/dupsno.f @@ -0,0 +1,648 @@ +C$$$ MAIN PROGRAM DOCUMENTATION BLOCK +C +C MAIN PROGRAM: BUFR_DUPSNO +C PRGMMR: MELCHIOR/DONG ORG: NP22 DATE: 2020-08-20 +C +C ABSTRACT: PROCESSES ANY COMBINATION UP TO SIX DUMP FILES CONTAINING +C TYPES THAT ULTIMATELY GO TO THE "ADPSFC" DUMP {BUFR MESSAGE +C TYPE 000, SUBTYPES 000 (SYNOPR), 001 (SYNOP), 002 (SYNOPM), +C 100 (SYNPBR, BUFR), 101 (SYNOPB, BUFR), 102 (SYNOMB, BUFR)}. +C PERFORMS A SINGLE DUP-CHECK FOR REPORTS ACROSS ALL APPLICABLE +C FILES GOING INTO THE "ADPSFC" DUMP. DOES NOT INCLUDE METAR IN +C SUBTYPE 007 WHICH GO INTO "ADPSFC" DUMP. INFORMATION IS READ +C SEPARATELY FROM EACH FILE THAT IS PRESENT, AND IS THEN +C COMBINED INTO TABLES USED FOR THE DUP-CHECK. THE ALGORITHM +C SORTS THE REPORTS IN ASCENDING ORDER OF LAT, LON, OBS TIME +C (DAY DOWN TO SECOND), HEIGHT(??), AND RECEIPT TIME (YEAR DOWN +C TO MINUTE). IN THE DUPLICATE CHECKING LOGIC, ADJACENT, SORTED +C REPORT PAIRS ARE CHECKED FOR LAT, LON, AND OBS TIME (TO THE +C SECOND), ALL BASED ON TOLERANCE LIMITS WHICH CAN VARY BASED ON +C WHETHER THIS IS A REAL-TIME OR HISTORICAL RUN. THE REPORT +C USUALLY SELECTED IS THE BULLETIN LAST RECEIVED, HOWEVER IF ONE +C REPORT IN THE PAIR IS IN TAC FORMAT, THE BUFR FORMAT (NON-TAC) +C IS ALWAYS SELECTED. THERE ARE OTHER RULES FOR SELECTING THE +C "BEST" REPORT IN THE DUPLICATE PAIR (SEE CODE). THE WORKING +C FILE NAMES OF THE INPUT DUMP FILES (IN EITHER THE "NEW" FORM +C x_ttt.sss, WHERE ttt IS BUFR TYPE, sss IS BUFR SUBTYPE, AND x IS AN +C ORDERING INDEX; OR THE "OLD" FORM ttt.sss) ARE READ FROM STANDARD +C INPUT (UNIT 5) AT THE START OF THIS PROGRAM. THE OUTPUT DUP-CHECKED +C FILES WILL BE WRITTEN TO THE SAME FILE NAMES. ALL OTHER FILE +C CONNECTIONS ARE MADE THROUGH THE FORTRAN OPEN STATEMENT. +C +C PROGRAM HISTORY LOG: +C 2020-08-20 S. MELCHIOR ORIGINAL VERSION FOR IMPLEMENTATION +C 2020-08-20 J. DONG -- +C - ADD A TEMPORARY SOLUTION TO KEEP TAC REPOERTS +C IF THE STATION HEIGHT IS MISSING IN THE BUFR REPORTS. +C - ADD A TEMPORARY SOLUTION TO KEEP TAC REPOERTS +C IF THE PMSL IS MISSING IN THE BUFR REPORTS. +C - PUT THE LAT/LON SORT PRIORITY BEHIND THE +C OBSERVATIONAL TIME AND RPID FOR CONSIDERING THE SLIGHT +C LAT/LON DIFFERENCE BETWEEN TAC AND BUFR FORMAT REPORTS +C IN THE DUPLICATION CHECKS. +C - ADDED SETBMISS CALL TO SET BMISS TO 10E8 AND +C CHANGE THE CODE TO AVOID INTEGER OVERFLOW +C 2021-04-20 X. Su adding statement for station with missing values +C 2021-04-20 J. DONG -- +C - CHANGED DEXY TO 0.005. +C - REMOVED THE UNNESSARY CODES +C - FIXED THE BUGS FOR KEEPING THE REPORTS WITH +C DIFFERENT IDs BUT WITH CLOSE LAT/LON. +C 2022-03-25 I. Genkova Added check for 0 reports in input file and +C allows for graceful continue in the event of 0 reports. +C +C USAGE: +C INPUT FILES: +C UNIT 05 - STANDARD INPUT - RECORDS CONTAINING THE WORKING INPUT +C FILE NAMES FOR ALL SYNOPTIC TYPES EVENTUALLY BEING +C COMBINED INTO A SINGLE DUMP FILE (for "adpsfc" dumps) +C - THE ONLY FILE NAMES CONSIDERED BY THIS PROGRAM ARE +C *000.000 (SYNOPR), *000.001 (SYNOP), +C *000.002 (SYNOPM), *000.100 (SYNPBR, BUFR), +C *000.101 (SYNOPB, BUFR), *000.102 (SYNPMB, BUFR) +C (all included in the "adpsfc" dump) +C - OTHER FILES MAY BE INCLUDED HERE, BUT THEY WILL +C NOT BE MODIFIED BY THIS PROGRAM; THE OUTPUT FILE +C NAMES WILL BE THE SAME AS THE INPUT NAMES HERE. +C UNIT 20 - UNCHECKED BUFR FILE(S) +C +C OUTPUT FILES: +C UNIT 20 - DUPLICATE CHECKED BUFR FILE(S) +C UNIT 50 - WORKSPACE (SCRATCH) FILE(S) +C UNIT 60 - STATUS FILE WHOSE PRESENCE INDICATES THIS PROGRAM +C COMPLETED SUCCESSFULLY +C +C SUBPROGRAMS CALLED: +C LIBRARY: +C W3NCO - W3TAGB W3TAGE ERREXIT IW3JDN +C W3EMC - ORDERS +C BUFRLIB - DATELEN OPENBF COPYMG UFBTAB OPENMB COPYBF STATUS +C COPYSB CLOSMG CLOSBF NEMTAB MESGBC IBFMS GETBMISS +C +C EXIT STATES: +C COND = 0 - SUCCESSFUL RUN +C > 0 - ABNORMAL RUN +C +C REMARKS: NONE. +C +C ATTRIBUTES: +C LANGUAGE: FORTRAN 90 +C MACHINE: WCOSS +C +C$$$ + PROGRAM BUFR_DUPSNO + + PARAMETER (MXTS=10) + PARAMETER (NFILES=6) ! Number of input files being considered + + REAL(8),ALLOCATABLE :: TAB_8(:,:) + real(8),allocatable :: rab_8(:,:) + INTEGER,ALLOCATABLE :: IWORK(:) + INTEGER,ALLOCATABLE :: IORD(:) + INTEGER,ALLOCATABLE :: JDUP(:),JJDUP(:) + + CHARACTER*80 TSTR,TSTH,rstr,FILE,FILI(NFILES),FILO + CHARACTER*8 SUBSET,CTAB7,ctab7_i,ctab7_j,c_missing + character*20 ctext_file(nfiles) + CHARACTER*3 DUMMY_MSGS + + REAL(8) UFBTAB_8,TAB7_8,BMISS,GETBMISS,tab7_i_8,tab7_j_8 + + DIMENSION IMST(0:102),ntab_file(nfiles) + DIMENSION NDUP(0:2),IPTR(2,NFILES),ndup_file(0:2,nfiles) + + LOGICAL DUPES,SYNOP + + EQUIVALENCE (TAB7_8,CTAB7) + equivalence (tab7_i_8,ctab7_i),(tab7_j_8,ctab7_j), + $ (c_missing,i_missing) + + DATA TSTR /'CLAT CLON MNTH DAYS HOUR MINU RPID SELV TOSD'/ + DATA TSTH /'CLATH CLONH MNTH DAYS HOUR MINU RPID HSMSL TOSD'/ + data rstr /'RCYR RCMO RCDY RCHR RCMI '/ + + data ctext_file/ + $ 'SYNOPR format ', + $ 'SYNOP format ', + $ 'SYNOPM format ', + $ 'SYNPBR/BUFR ', + $ 'SYNOPB/BUFR ', + $ 'SYNPMB/BUFR '/ + + data c_missing/'missing-'/ + +C Tolerance parameters for all reports: +C (exact for the moment, but lat/lon and time might need some +C tolearnce) +C ------------------------------------- + + data dexy / 0.005/ ! lat/lon + data dmon / 0.0/ ! month + DATA DDAY / 0.0/ ! day + data dour / 0.0/ ! hour + data dmin / 0.0/ ! minute + +C IMST (index for message subtype) - must start at 0 rather than +C default of 1 because first eligible synop tank is b000/xx000. + DATA IMST / 1, 2, 3, 97*-99, 4, 5, 6/ + +C----------------------------------------------------------------------- +C----------------------------------------------------------------------- + CALL W3TAGB('BUFR_DUPSNO',2022,0084,0054,'NP22') + + print * + print * ,'---> Welcome to BUFR_DUPSNO - Version 05-22-2024' + print * + + CALL DATELEN(10) + +ccccc CALL OPENBF(0,'QUIET',2) ! Uncomment for extra print from bufrlib + +C ASSIGN DEFAULT VALUE FOR 'MISSING' TO LOCAL BMISS VARIABLE +C ---------------------------------------------------------- + + CALL SETBMISS(10E8_8) + BMISS = GETBMISS() ! assign default value for "missing" + print * + print *, 'BUFRLIB value for missing is: ',bmiss + print * + +C SET THE COUNTERS TO INITIAL VALUES +C ---------------------------------- + + LUBFI = 20 + LUBFJ = 50 + +C STORE THE FILENAMES TO PROCESS FROM STANDARD INPUT +C -------------------------------------------------- + + FILI(1:NFILES)(1:4) = 'NONE' + + SYNOP = .FALSE. + +1 CONTINUE + + READ(5,'(A80)',END=2) FILE + DO I=1,10 + IF(FILE(I:I+3).EQ.'000.') THEN + READ(FILE(I+4:I+6),'(I3)') MST + IF(MST.LE.2.OR.MST.EQ.100.or.mst.eq.101.OR. + . MST.EQ.102) THEN + FILI(IMST(MST)) = FILE + SYNOP = .TRUE. + PRINT *, ' >> WILL CHECK ',FILE(I:I+6) + ENDIF + EXIT + ENDIF + ENDDO + GOTO 1 + +2 CONTINUE + IF(.NOT.SYNOP) THEN + PRINT * + PRINT *,'BUFR_DUPSNO: NO SYNOPTIC REPORTS TO CHECK' + PRINT * + CALL W3TAGE('BUFR_DUPSNO') + STOP + ELSE + print 200, dexy,dday,dour + 200 FORMAT(/'BUFR_DUPSNO PARAMETERS:'/ + . 3X,'TOLERANCE FOR LAT/LON CHECKS (IN DEGREES) .. ',F7.3/ + . 3X,'TOLERANCE FOR DAY CHECK (IN DAYS) .......... ',F7.3/ + . 3X,'TOLERANCE FOR HOUR CHECK (IN HOURS) ........ ',F7.3/) + ENDIF + +C COUNT THE NUMBER OF SUBSETS AMONGST ALL FILES TO ALLOCATE SPACE +C --------------------------------------------------------------- + + MXTB = 0 + DO I=1,NFILES + IF(FILI(I)(1:4).EQ.'NONE') CYCLE + CALL CLOSBF(LUBFI) + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + CALL OPENBF(0,'QUIET',1) ! will generate diagnostic print if + ! an embedded BUFR table is read + CALL UFBTAB(-LUBFI,UFBTAB_8,1,1,NUM_SUBSETS,' ') + CALL OPENBF(0,'QUIET',0) ! return to default wrt degree of prnt + MXTB = MXTB + NUM_SUBSETS + ENDDO + + ISUB = 0 + NTAB = 0 + + IF(MXTB.EQ.0) THEN + PRINT * + PRINT *, '### WARNING: A total of ZERO input synoptic reports' + PRINT * + GO TO 400 + ENDIF + + ALLOCATE(TAB_8(MXTS,MXTB),STAT=I);IF(I.NE.0) GOTO 901 + allocate(rab_8(mxts,mxtb),stat=i);if(i.ne.0) goto 901 + ALLOCATE(IWORK(MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 + ALLOCATE(IORD(MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 + ALLOCATE(JDUP(MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 + ALLOCATE(JJDUP(MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 + + TAB_8 = BMISS + rab_8 = bmiss + +C MAKE A TABLE OUT OF THE LATS, LONS, AND TIME COORDINATES +C -------------------------------------------------------- + + IPTR = 0 + IPT = 1 + + DO I=1,NFILES + IF(FILI(I)(1:4).NE.'NONE') THEN + CALL CLOSBF(LUBFI) + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + + CALL MESGBC(LUBFI,MSGT,ICOMP) + IF(ICOMP.EQ.1) THEN + PRINT'(/"INPUT BUFR FILE",I2," MESSAGES '// + . 'C O M P R E S S E D"/"FIRST MESSAGE TYPE FOUND IS",'// + . 'I5/)', I,MSGT + PRINT'("#####BUFR_DUPSNO (UFBTAB) CANNOT PROCESS '// + . 'COMPRESSED BUFR MESSAGES -- FATAL ERROR")' + CALL W3TAGE('BUFR_DUPSNO') + CALL ERREXIT(99) + ELSE IF(ICOMP.EQ.0) THEN + PRINT'(/"INPUT BUFR FILE",I2," MESSAGES '// + . 'U N C O M P R E S S E D"/"FIRST MESSAGE TYPE FOUND '// + . 'IS",I5/)', I,MSGT + ELSE IF(ICOMP.EQ.-1) THEN + PRINT'(//"ERROR READING INPUT BUFR FILE",I2," - '// + . 'MESSAGE COMPRESSION UNKNOWN"/)', I + ELSE IF(ICOMP.EQ.-3) THEN + PRINT'(/"INPUT BUFR FILE",I2," DOES NOT EXIST"/)', I + ELSE IF(ICOMP.EQ.-2) THEN + PRINT'(/"INPUT BUFR FILE",I2," HAS NO DATA MESSAGES"/'// + . '"FIRST MESSAGE TYPE FOUND IS",I5/)', I,MSGT + ENDIF + CALL UFBTAB(LUBFI,TAB_8(1,IPT),MXTS,MXTB-IPT+1,NTAB,TSTR) + IF(IBFMS(TAB_8(2,IPT)).EQ.1) THEN ! CLON missing + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + CALL UFBTAB(LUBFI,TAB_8(1,IPT),MXTS,MXTB-IPT+1,NTAB,TSTH) + ENDIF + call ufbtab(lubfi,rab_8(1,ipt),mxts,mxtb-ipt+1,ntab,rstr) + IPTR(1,I) = IPT + IPTR(2,I) = IPT+NTAB-1 + IPT = IPT+NTAB + ENDIF + ENDDO + + NTAB = IPT-1 + +C The arrays containing receipt time coordinates are set to negative +C values so that the key on them in sort routine ORDERS below will +C order them from latest actual receipt time to earliest - this is +C needed because logic is designed to normally toss the second report +C in a duplicate pair and we want to retain the one with the latest +C receipt time. +C -------------------------------------------------------------------- + + rab_8 = -rab_8 + +C INITIAL VALUES FOR MARKERS AND COUNTERS +C --------------------------------------- + + JDUP = 0 + JJDUP = 0 + IORD = 0 + NDUP = 0 + ndup_file = 0 + ntab_file = 0 + +C SET MISSING MINU TO ZERO; STORE INPUT FILE INDEX +C ------------------------------------------------ + + DO N=1,NTAB + + IF(IBFMS(TAB_8(6,N)).EQ.1) then + TAB_8(6,N) = 0 ! data missing + endif + + ! Store input file index in 8th index + ! TAB_8(10,N) = 1 --> b000/xx000 (SYNOPR) + ! = 2 --> b000/xx001 (SYNOP) + ! = 3 --> b000/xx002 (SYNOPM) + ! = 4 --> b000/xx100 (SYNPBR/BUFR) + ! = 5 --> b000/xx101 (SYNOPB/BUFR) + ! = 6 --> b000/xx102 (SYNPMB/BUFR) + + DO I=1,NFILES + IF(N.GE.IPTR(1,I) .AND. N.LE.IPTR(2,I)) THEN + TAB_8(10,N) = I + ntab_file(i) = ntab_file(i) + 1 + EXIT + ENDIF + ENDDO + + ENDDO ! end DO N=1,NTAB + +C GET A SORTED INDEX OF THE REPORTS KEYED IN THIS ORDER: +C RECEIPT TIME, LAT/LON, RPID, OBS TIME +C ---------------------------------------------------------------- + + call orders( 2,iwork,rab_8(5,1),iord,ntab,mxts,8,2) ! rcpt minute + call orders(12,iwork,rab_8(4,1),iord,ntab,mxts,8,2) ! rcpt hour + call orders(12,iwork,rab_8(3,1),iord,ntab,mxts,8,2) ! rcpt day + call orders(12,iwork,rab_8(2,1),iord,ntab,mxts,8,2) ! rcpt month + CALL ORDERS(12,IWORK,RAB_8(1,1),IORD,NTAB,MXTS,8,2) ! rcpt year + CALL ORDERS(12,IWORK,TAB_8(2,1),IORD,NTAB,MXTS,8,2) ! lon + CALL ORDERS(12,IWORK,TAB_8(1,1),IORD,NTAB,MXTS,8,2) ! lat + CALL ORDERS(10,IWORK,TAB_8(7,1),IORD,NTAB,MXTS,8,2) ! rpid + CALL ORDERS(12,IWORK,TAB_8(6,1),IORD,NTAB,MXTS,8,2) ! minute + CALL ORDERS(12,IWORK,TAB_8(5,1),IORD,NTAB,MXTS,8,2) ! hour + CALL ORDERS(12,IWORK,TAB_8(4,1),IORD,NTAB,MXTS,8,2) ! day + CALL ORDERS(12,IWORK,TAB_8(3,1),IORD,NTAB,MXTS,8,2) ! month + +C GO THROUGH THE REPORTS IN ORDER, MARKING DUPLICATES +C --------------------------------------------------- + + LOOP1: DO L=1,NTAB-1 + K = L+1 + I = IORD(L) + J = IORD(L+1) + tab7_i_8 = tab_8(7,i) ! rpid + tab7_j_8 = tab_8(7,j) ! rpid + if(jdup(i).ne.0) then + print *, '---> this report in record (I) has already been ', + . 'flagged as a duplicate - cycle' + cycle LOOP1 + endif + + DO WHILE(JDUP(I).EQ.0) + if(jdup(j).ne.0) then + print *, '---> this report in record (J) has already ', + . 'been flagged as a duplicate - cycle' + go to 800 + endif + dell = dexy + JJDUP(j)=0 + if(ctab7_i .eq. ctab7_j ) then + if(NINT(ABS(TAB_8(3,I)-TAB_8(3,J))*100.).LE. + . NINT(DMON*100.).AND. + . NINT(ABS(TAB_8(4,I)-TAB_8(4,J))*100.).LE. + . NINT(DDAY*100.).AND. + . NINT(ABS(TAB_8(5,I)-TAB_8(5,J))*100.).LE. + . NINT(DOUR*100.).AND. + . NINT(ABS(TAB_8(6,I)-TAB_8(6,J))*100.).LE. + . NINT(DMIN*100.)) then +cc print 1799, i,ctab7_i,(tab_8(ii,i),ii=1,2), +cc . (nint(tab_8(ii,i)),ii=4,6), +cc . (nint(-rab_8(ii,i)),ii=1,5),nint(tab_8(10,i)) +cc . ,tab_8(8,i),tab_8(9,i) +cc 1799 format('===> DUPL. FOUND:'/' --> I: ',I6,';{ ID: ',A8, +cc . '; LAT: ',F9.5,'; LON: ',F10.5,'; RPRT DD HH:mm ', +cc . I2.2,I2,':',I2,'; RCPT YYYYMMDDHHMM: ',I4,4I2.2, +cc . '; file #',I2,'} ELEV=',F15.2,'TOSD=',F15.5) +cc print 1800, j,ctab7_j,(tab_8(ii,j),ii=1,2), +cc . (nint(tab_8(ii,j)),ii=4,6), +cc . (nint(-rab_8(ii,j)),ii=1,5),nint(tab_8(10,j)) +cc . ,tab_8(8,j),tab_8(9,j) +cc 1800 format(' --> J: ',I6,';{ ID: ',A8,'; LAT: ',F9.5, +cc . '; LON: ',F10.5,'; RPRT',' DD HH:mm ',I2.2,I2,':',I2, +cc . '; RCPT YYYYMMDDHHMM: ',I4,4I2.2, +cc . '; file #',I2,'} ELEV=',F15.2,'TOSD=',F15.5) + +cc print *,'lat diff =',ABS(TAB_8(1,I)-TAB_8(1,J)) +cc print *,'lon diff =',ABS(TAB_8(2,I)-TAB_8(2,J)) + ! when reports "I" and "J" satisfy the duplicate + ! check, the default is to set report "J"'s + ! duplicate flag to 1 in order to throw it out + ! (thus retaining report "I") - however this is + ! overridden in the following cases {whereby + ! report "I"'s duplicate flag is set to 1 in order + ! to throw it out (thus instead retaining report "J")}: + JJDUP(j)=1 + endif + endif + if(JJDUP(J) .EQ.1) then + if(ctab7_i.eq.'missing-') then + ! case 1: report "I" has a missing report identifier + jdup(i) = 1 + print *, 'I tossed ==> missing report id' + else if(ctab7_j.ne.'missing-' + . .and. (tab_8(8,j).lt.bmiss) + . .and. ((tab_8(9,i).ge.bmiss).or.(tab_8(9,j).lt.bmiss)) + . .and. (tab_8(10,i).le.3.and.tab_8(10,j).ge.4)) then + ! case 2: report "I" is from a less-preferred + ! file-type than report "J" {e.g., BUFR format + ! restricted data, SYNPBR preferred over TAC format + ! restricted data, SYNOPR; SYNOPB preferred over + ! SYNOP; SYNPMB preferred over SYNOPM} + ! and report "J" has a non-missing report identifier + jdup(i) = 1 + print *, 'I tossed ==> from less-preferred file-type', + . ' than J' + print *,'I,J:',nint(tab_8(10,i)),nint(tab_8(10,j)) + else if( (tab_8(8,j).lt.bmiss) + . .and. (tab_8(9,i).ge.bmiss.and.tab_8(9,j).lt.bmiss) + . .and. (tab_8(10,i).ge.4.and.tab_8(10,j).le.3)) then + ! case 3: report "J" is from a less-preferred + ! file-type than report "I" (see case 2), But, + ! report "J" has a non-missing TOSD and ELEV + ! report "I" has a missing TOSD or ELEV. + jdup(i) = 1 + print *, 'I tossed ==> from missing ELEV or TOSD', + . ' in report I' + print *,'I,J:',nint(tab_8(10,i)),nint(tab_8(10,j)) + end if + ! At this point, if report "I"'s duplicate flag is not + ! set in one of the above cases, set report "J"'s + ! duplicate flag to 1 in order to throw it out thus + ! retaining report "I" + IF(JDUP(I).EQ.0) then + JDUP(J) = 1 + print *, 'J tossed' + print *,'I,J:',nint(tab_8(10,i)),nint(tab_8(10,j)) + END IF + + ENDIF ! endif for dupe test/comparison + JJDUP(J)=0 + 800 continue + IF(K+1.GT.NTAB) CYCLE LOOP1 + J = IORD(K+1) + tab7_j_8 = tab_8(7,j) + K = K+1 + ENDDO + ENDDO LOOP1 + +C WRITE BACK THE DUP-CHECKED FILE(S) +C ---------------------------------- + + CALL GETENV('DUMMY_MSGS',DUMMY_MSGS) + + DO I=1,NFILES + IF(FILI(I)(1:4).NE.'NONE') THEN +cpppppppppp +cc idate_last = -9999 +cpppppppppp + FILO = '.'//FILI(I) + ISUB = IPTR(1,I)-1 + CALL CLOSBF(LUBFI) + CALL CLOSBF(LUBFJ) + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + OPEN(LUBFJ,FILE=FILO ,FORM='UNFORMATTED') + CALL OPENBF(LUBFI,'IN ',LUBFI) + CALL OPENBF(LUBFJ,'OUT',LUBFI) + +C If input file doesn't contain dummy center and dump time messages 1 +C and 2 (after table messages), before doing anything call closmg +C with a negative unit number to signal routine that it should not +C write out ANY messages with zero subsets in them - this holds for +C all subsequent calls to closmg in this routine, even those done +C through other bufrlib routines (and even for those calls where the +C sign of the unit number is positive) +C -------------------------------------------------------------------- + + IF(DUMMY_MSGS.NE.'YES') CALL CLOSMG(-LUBFJ) + + DO WHILE(IREADMG(LUBFI,SUBSET,IDATE).EQ.0) + NSUBS = NMSUB(LUBFI) +cpppppppppp +cc print *, 'New message read in , NSUBS, IDATE = ',nsubs,idate +cpppppppppp + +C If no subsets in msg & dummy msgs not expected loop to next input msg +C --------------------------------------------------------------------- + + IF(NSUBS.LE.0.AND.DUMMY_MSGS.NE.'YES') CYCLE + + DUPES = .FALSE. + + IF(NSUBS.GT.0) THEN + DO N=1,NSUBS + IF(ISUB+N.GT.NTAB) THEN + IDUP = 4 + ELSE + IDUP = JDUP(ISUB+N) + ENDIF + NDUP(IDUP) = NDUP(IDUP)+1 + ndup_file(idup,i) = ndup_file(idup,i)+1 + IF(IDUP.GT.0) DUPES = .TRUE. + ENDDO + ENDIF + +cpppppppppp +cc print *, 'DUPES = ',dupes +cpppppppppp + + IF(DUPES) THEN + CALL OPENMB(LUBFJ,SUBSET,IDATE) +cpppppppppp +cc if(idate.ne.idate_last) +cc $ print *, 'NEW MESSAGE OPENED' +cc idate_last = idate +cpppppppppp + DO WHILE(IFBGET(LUBFI).EQ.0) + ISUB = ISUB+1 + IF(ISUB.GT.NTAB) THEN + IDUP = 4 + ELSE + IDUP = JDUP(ISUB) + ENDIF + IF(IDUP.EQ.0) THEN + CALL COPYSB(LUBFI,LUBFJ,IRET) ! Copy non-dups + ELSE + CALL COPYSB(LUBFI,00000,IRET) ! Toss dups + ENDIF + ENDDO + ELSE + +C In the event that the input file contains dummy center and dump time +C messages 1 and 2 (after table messages), call closmg with a +C positive unit number to signal routine that it should write out +C these messages even though they have zero subsets in them +C If the input file does not contain dummy messages, a positive unit +C number here is immaterial because closmg was already called with +C a negative unit number immediately after the output file was +C opened (and this holds for all subsequent calls to closmg +C regardless of the sign of the unit number) +C ------------------------------------------------------------------- + + CALL CLOSMG(LUBFJ) + CALL COPYMG(LUBFI,LUBFJ) ! Copy non-dups + ISUB = ISUB+NSUBS + ENDIF + ENDDO + + CALL CLOSBF(LUBFI) + CALL CLOSBF(LUBFJ) + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + OPEN(LUBFJ,FILE=FILO ,FORM='UNFORMATTED') + CALL COPYBF(LUBFJ,LUBFI) + OPEN(LUBFI,FILE=FILI(I),FORM='UNFORMATTED') + + CALL MESGBC(LUBFI,MSGT,ICOMP) + IF(ICOMP.EQ.1) THEN + PRINT'(/"OUTPUT BUFR FILE",I2," MESSAGES '// + . 'C O M P R E S S E D"/"FIRST MESSAGE TYPE FOUND IS",'// + . 'I5/)', I,MSGT + ELSE IF(ICOMP.EQ.0) THEN + PRINT'(/"OUTPUT BUFR FILE",I2," MESSAGES '// + . 'U N C O M P R E S S E D"/"FIRST MESSAGE TYPE FOUND '// + . 'IS",I5/)', I,MSGT + ELSE IF(ICOMP.EQ.-1) THEN + PRINT'(//"ERROR READING OUTPUT BUFR FILE",I2," - '// + . 'MESSAGE COMPRESSION UNKNOWN"/)', I + ELSE IF(ICOMP.EQ.-3) THEN + PRINT'(/"OUTPUT BUFR FILE",I2," DOES NOT EXIST"/)', I + ELSE IF(ICOMP.EQ.-2) THEN + PRINT'(/"OUTPUT BUFR FILE",I2," HAS NO DATA MESSAGES"/'// + . '"FIRST MESSAGE TYPE FOUND IS",I5/)', I,MSGT + ENDIF + + CLOSE(LUBFI) + ENDIF + ENDDO + + 400 CONTINUE + +C GENERATE REPORT SUMMARY +C ----------------------- + + PRINT 300, ISUB,NDUP(2),MXTB,NTAB + 300 FORMAT(/'BUFR_DUPSNO READ IN A TOTAL OF',I8,' REPORTS'/ + . ' A TOTAL OF ',I7,' REPORTS WERE SKIPPED DUE TO BEING ', + . 'OVER THE LIMIT OF ',I7// + . 'BUFR_DUPSNO CHECKED A TOTAL OF',I8,' REPORTS') + do i = 1,nfiles + if(ntab_file(i).gt.0) print 301, ntab_file(i),ctext_file(i) + 301 format(' THESE INCLUDE ',I8,' REPORTS FROM ',A20) + enddo + + print 302, ndup(0) + 302 format(/'NUMBER OF UNIQUE REPORTS WRITTEN OUT ...........',i7) + do i = 1,nfiles + if(ndup_file(0,i).gt.0) print 303, ndup_file(0,i),ctext_file(i) + enddo + 303 format(10x,'THESE INCLUDE ',i8,' REPORTS FROM ',a20) + + print 304, ndup(1) + 304 format(/'NUMBER OF REPORTS SKIPPED DUE TO:'/ + . ' FAILING DUPLICATE CHECK (ALL SOURCES) .......',i7) + do i = 1,nfiles + if(ndup_file(1,i).gt.0) print 303, ndup_file(1,i),ctext_file(i) + enddo + +C END OF PROGRAM +C -------------- + + WRITE(60,'("ALL DONE")') + CALL W3TAGE('BUFR_DUPSNO') + STOP + +C ERROR EXITS +C ----------- + + 901 CONTINUE + + PRINT *, '#####BUFR_DUPSNO - UNABLE TO ALLOCATE ARRAYS' + CALL W3TAGE('BUFR_DUPSNO') + CALL ERREXIT(99) + + END + diff --git a/sorc/bufr_dupupr.fd/dupupr.f b/sorc/bufr_dupupr.fd/dupupr.f index 6f4c88c..4dc96cf 100644 --- a/sorc/bufr_dupupr.fd/dupupr.f +++ b/sorc/bufr_dupupr.fd/dupupr.f @@ -513,7 +513,7 @@ PROGRAM BUFR_DUPUPR C-----IG_TIME_sections_of_this_code call cpu_time(TT8) - IF(0.EQ.0) THEN ! IG TEST 2 start + IF(0.EQ.1) THEN ! IG TEST 2 start !turn off CORN rewrite print * ,'ILIANA in second slow down' OPEN(LUBFI,FILE=FILI(1:NBYTES_FILI),FORM='UNFORMATTED') CALL UFBTAB(LUBFI,RAB_8,MXTS,MXTB,NTAB,RSTR) @@ -554,7 +554,7 @@ PROGRAM BUFR_DUPUPR print * ,'ILIANA= runs the MINIMUM (6) out of 13 ORDERS' CALL ORDERS( 2,IWORK,TAB_8(7,1),IORD,NTAB,MXTS,8,2) ! correction -c IG skip some sorting + !IG skip unnecessary sorting by receipt time !CALL ORDERS(12,IWORK,RAB_8(5,1),IORD,NTAB,MXTS,8,2) ! rcpt minute !CALL ORDERS(12,IWORK,RAB_8(4,1),IORD,NTAB,MXTS,8,2) ! rcpt hour !CALL ORDERS(12,IWORK,RAB_8(3,1),IORD,NTAB,MXTS,8,2) ! rcpt day diff --git a/sorc/bufr_geofil.fd/geofil.f b/sorc/bufr_geofil.fd/geofil.f index 2056331..b30561c 100644 --- a/sorc/bufr_geofil.fd/geofil.f +++ b/sorc/bufr_geofil.fd/geofil.f @@ -315,10 +315,10 @@ PROGRAM BUFR_GEOFIL OPEN(LUBFI,FILE=FILI(1:NBYTES_FILI),FORM='UNFORMATTED') CALL OPENBF(0,'QUIET',1) ! will generate diagnostic print if an ! embedded BUFR table is read - CALL MAXOUT(70000) !IG Try to fix CPYUPD ABORT + CALL MAXOUT(80000) !IG Try to fix CPYUPD ABORT CALL UFBTAB(-LUBFI,UFBTAB_8,1,1,MXTB,' ') CALL OPENBF(0,'QUIET',0) ! return to default wrt degree of print - CALL MAXOUT(70000) !IG Try to fix CPYUPD ABORT + CALL MAXOUT(80000) !IG Try to fix CPYUPD ABORT ALLOCATE(TAB_8(MXTS,MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 ALLOCATE(JGEO(MXTB) ,STAT=I);IF(I.NE.0) GOTO 901 diff --git a/ush/bufr_dump_obs.sh b/ush/bufr_dump_obs.sh index 9ea3172..9b6b0ee 100755 --- a/ush/bufr_dump_obs.sh +++ b/ush/bufr_dump_obs.sh @@ -1562,7 +1562,7 @@ do errmvl=$errmv errmv=`expr $errmvl + $errmvt` else - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}${n}.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}${n}.${tmmark}.bufr_d fi if [ "$CHGRP_RSTPROD" = 'YES' ]; then @@ -1583,7 +1583,7 @@ rstprod group have read permission" echo " " >> outout echo " " >> status1.out else - cp /dev/null ${COMSP}${n}.${tmmark}.bufr_d + cpfs /dev/null ${COMSP}${n}.${tmmark}.bufr_d msg="**WARNING: $n dump contains RESTRICTED data, since user \ $USER is not in rstprod group a null file is copied in its place" set +u @@ -1727,8 +1727,8 @@ done [ "$kflag" -eq '0' ] && kindx=3 -cp /dev/null sstgrb -cp /dev/null sstgrb.index +cpfs /dev/null sstgrb +cpfs /dev/null sstgrb.index errsmi=0 msg1=NO @@ -1736,7 +1736,7 @@ msg1=NO for nnn do [ "$nnn" = $ssmif -a "$SENDCOM" != 'NO' ] && \ - cp /dev/null ${COMSP}spssmi.${tmmark}.bufr_d + cpfs /dev/null ${COMSP}spssmi.${tmmark}.bufr_d if [ "$nnn" = $ssmif -a "$prepssmi" = 'YES' ];then set +x @@ -1770,14 +1770,14 @@ do cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}$ssmif.${tmmark}.bufr_d ssmif_in + cpfs ${COMSP}$ssmif.${tmmark}.bufr_d ssmif_in SSMI_IN=ssmif_in ssmiOU=${COMSP}spssmi.${tmmark}.bufr_d if [ "$PREPSSMI_PROD_TYPE" = 'COMBINATION' ];then ssmiOU_save=$ssmiOU ssmiOU=$ssmiOU_save.$ssmif - cp /dev/null $ssmiOU + cpfs /dev/null $ssmiOU fi DATA_save=$DATA @@ -1811,7 +1811,7 @@ do [ -s msg1_file ] && msg1="`cat msg1_file` from input $ssmif dump" [ -s msg2_file ] && msg2="`cat msg2_file` from input $ssmif dump" - cp ssmi.bufr $ssmiOU + cpfs ssmi.bufr $ssmiOU errmvt=$? errmvl=$errmv errmv=`expr $errmvl + $errmvt` @@ -1879,8 +1879,8 @@ set -u # ------------------------------------------------------------------ if [ -s $ssmiOU_save.ssmipn -a -s $ssmiOU_save.ssmip ];then - cp $ssmiOU_save.ssmipn bufr1 - cp $ssmiOU_save.ssmip bufr2 + cpfs $ssmiOU_save.ssmipn bufr1 + cpfs $ssmiOU_save.ssmip bufr2 cat < cards bufr1 bufr2 @@ -1935,18 +1935,18 @@ generated" echo " " >> $pgmout_this [ "$err" -gt '0' ] && exit 99 elif [ -s $ssmiOU_save.ssmipn ];then - cp $ssmiOU_save.ssmipn combfr.spssmi + cpfs $ssmiOU_save.ssmipn combfr.spssmi err=$? elif [ -s $ssmiOU_save.ssmip ];then - cp $ssmiOU_save.ssmip combfr.spssmi + cpfs $ssmiOU_save.ssmip combfr.spssmi err=$? else - cp /dev/null combfr.spssmi + cpfs /dev/null combfr.spssmi err=$? fi rm $ssmiOU_save.ssmipn $ssmiOU_save.ssmip if [ $err -eq 0 ];then - cp combfr.spssmi $ssmiOU_save + cpfs combfr.spssmi $ssmiOU_save if [ -s ssmi_count ];then n1=`head -n1 ssmi_count` n2=`tail -n1 ssmi_count` @@ -1988,7 +1988,7 @@ do msg1=NO msg2="**NO ERS SCATTEROMETER DATA reprocessed due to above PGM FAIL \ --> non-fatal" - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}erscat.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}erscat.${tmmark}.bufr_d if [ "$prepersd" = 'YES' ];then @@ -2002,7 +2002,7 @@ echo "*******************************************************************\ cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}erswnd.${tmmark}.bufr_d erswnd_bufr + cpfs ${COMSP}erswnd.${tmmark}.bufr_d erswnd_bufr ersIN=erswnd_bufr ersOU=${COMSP}erscat.${tmmark}.bufr_d @@ -2259,7 +2259,7 @@ echo "------------------------------------------------------------" WAVE_DATASORT all completed normally" ########################cp ers2.so ${COMSP}ers2.sort - cp erscat_bufr $ersOU + cpfs erscat_bufr $ersOU errcpy=$? if [ "$errcpy" -eq '0' ];then msg2="ERS scatterometer data reprocessed successfully into NCEP BUFR file" @@ -2340,7 +2340,7 @@ do msg2="**NO QUIKSCAT SCATTEROMETER DATA reprocessed (from input \ qkscat dump) due to above PGM FAIL --> non-fatal" - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}qkswnd.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}qkswnd.${tmmark}.bufr_d if [ "$prepqksd" = 'YES' ];then @@ -2357,7 +2357,7 @@ qkscat dump) due to above PGM FAIL --> non-fatal" cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}qkscat.${tmmark}.bufr_d qkscat_bufr + cpfs ${COMSP}qkscat.${tmmark}.bufr_d qkscat_bufr qksIN=qkscat_bufr qksOU=${COMSP}qkswnd.${tmmark}.bufr_d @@ -2391,7 +2391,7 @@ qkscat dump) due to above PGM FAIL --> non-fatal" # ----------------------------------------------------------------- if [ -s $DQKC ]; then - cp $DQKC dcodquikscat.parm + cpfs $DQKC dcodquikscat.parm else cat < dcodquikscat.parm &RDATA @@ -2460,7 +2460,7 @@ echo "------------------------------------------------------------" msg1="program WAVE_DCODQUIKSCAT completed normally, qkswnd \ file generated from input qkscat dump" - cp qkswnd_bufr $qksOU + cpfs qkswnd_bufr $qksOU errcpy=$? msg2=NO [ "$errcpy" -ne '0' ] && msg2="**NO QUIKSCAT SCATTEROMETER \ @@ -2536,7 +2536,7 @@ do msg2="**NO ASCAT SCATTEROMETER DATA reprocessed (from input ascatt \ dump) due to above PGM FAIL --> non-fatal" - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}ascatw.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}ascatw.${tmmark}.bufr_d if [ "$prepascd" = 'YES' ];then @@ -2553,7 +2553,7 @@ dump) due to above PGM FAIL --> non-fatal" cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}ascatt.${tmmark}.bufr_d ascatt_bufr + cpfs ${COMSP}ascatt.${tmmark}.bufr_d ascatt_bufr ascIN=ascatt_bufr ascOU=${COMSP}ascatw.${tmmark}.bufr_d @@ -2587,7 +2587,7 @@ dump) due to above PGM FAIL --> non-fatal" # ----------------------------------------------------------------- if [ -s $DASC ]; then - cp $DASC dcodascat.parm + cpfs $DASC dcodascat.parm else cat < dcodascat.parm &RDATA @@ -2657,7 +2657,7 @@ echo "------------------------------------------------------------" msg1="program WAVE_DCODQUIKSCAT completed normally, ascatw \ file generated from input ascatt dump" - cp ascatw_bufr $ascOU + cpfs ascatw_bufr $ascOU errcpy=$? msg2=NO [ "$errcpy" -ne '0' ] && msg2="**NO ASCAT SCATTEROMETER DATA \ @@ -2732,7 +2732,7 @@ do msg2="**NO TRMM TMI DATA reprocessed (from input trmm dump) due to \ above PGM FAIL --> non-fatal" - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}sptrmm.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}sptrmm.${tmmark}.bufr_d if [ "$preptrmm" = 'YES' ];then @@ -2749,7 +2749,7 @@ status1.out cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}trmm.${tmmark}.bufr_d trmm_bufr + cpfs ${COMSP}trmm.${tmmark}.bufr_d trmm_bufr tmiIN=trmm_bufr tmiOU=${COMSP}sptrmm.${tmmark}.bufr_d @@ -2780,7 +2780,7 @@ status1.out # ----------------------------------------------------------------- if [ -s $DTMC ]; then - cp $DTMC supertmi.parm + cpfs $DTMC supertmi.parm else cat < supertmi.parm &INPUT @@ -2839,7 +2839,7 @@ echo "------------------------------------------------------------" msg1="program BUFR_SUPERTMI completed normally, sptrmm file \ generated from input trmm dump" - cp sptrmm_bufr $tmiOU + cpfs sptrmm_bufr $tmiOU errcpy=$? msg2=NO [ "$errcpy" -ne '0' ] && msg2="**NO TRMM TMI DATA reprocessed \ @@ -2914,7 +2914,7 @@ do msg2="**NO WINDSAT SCATTEROMETER DATA reprocessed (from input wndsat \ dump) due to above PGM FAIL --> non-fatal" - [ "$SENDCOM" != 'NO' ] && cp /dev/null ${COMSP}wdsatr.${tmmark}.bufr_d + [ "$SENDCOM" != 'NO' ] && cpfs /dev/null ${COMSP}wdsatr.${tmmark}.bufr_d if [ "$prepwindsat" = 'YES' ];then @@ -2931,7 +2931,7 @@ dump) due to above PGM FAIL --> non-fatal" cd $DATA/job${DUMP_NUMBER} - cp ${COMSP}wndsat.${tmmark}.bufr_d wndsat_bufr + cpfs ${COMSP}wndsat.${tmmark}.bufr_d wndsat_bufr wnsIN=wndsat_bufr wnsOU=${COMSP}wdsatr.${tmmark}.bufr_d @@ -2965,7 +2965,7 @@ dump) due to above PGM FAIL --> non-fatal" # ---------------------------------------------------------------- if [ -s $DWSC ]; then - cp $DWSC dcodwindsat.parm + cpfs $DWSC dcodwindsat.parm else cat < dcodwindsat.parm &RDATA @@ -3026,7 +3026,7 @@ echo "------------------------------------------------------------" msg1="program BUFR_DCODWINDSAT completed normally, wdsatr file \ generated from input wndsat dump" - cp wdsatr_bufr $wnsOU + cpfs wdsatr_bufr $wnsOU errcpy=$? msg2=NO [ "$errcpy" -ne '0' ] && msg2="**NO WINDSAT SCATTEROMETER \ @@ -3104,15 +3104,16 @@ EOFblank > updated_counts.out rm status2.out - grep -e "Dumping [0-2]" -e "Missing [0-2]" $pgmout_this | cut -f2- -d" " \ + #tr -cd '[:print:]\n\r' < $pgmout_this > $pgmout_this #IG + grep --text -e "Dumping [0-2]" -e "Missing [0-2]" $pgmout_this | cut -f2- -d" " \ > cutLv.allout - grep "^....... HAS" $pgmout_this | cut -f2- -d" " > cutRv.allout + grep --text "^....... HAS" $pgmout_this | cut -f2- -d" " > cutRv.allout paste -d"\0\n" cutLv.allout cutRv.allout > pasteB.allout cut -c1-64 pasteB.allout > cutLv.allout cut -c65- pasteB.allout > cutRv.allout rm pasteB.allout paste -d"\0" cutLv.allout cutRv.allout > paste.part1 - grep -e "Domain for [0-2]" $pgmout_this > paste.part2 + grep --text -e "Domain for [0-2]" $pgmout_this > paste.part2 paste -d"\n\n" paste.part1 paste.part2 blank > counts.out nindx=`cat temp1 - grep -e "in data group" temp1 | grep -v -e "Domain" + grep --text -e "in data group" temp1 | grep -v -e "Domain" err_grep=$? if [ $err_grep -ne 0 ]; then cat temp1 >> updated_counts.out @@ -3270,7 +3271,8 @@ cat <<\EOFs2 >> status.out ############################################################################### EOFs2 - grep -q -e " SAT. ID " -e " %#" $pgmout + #tr -cd '[:print:]\n\r' < $pgmout > $pgmout #IG + grep --text -q -e " SAT. ID " -e " %#" $pgmout errgrep=$? if [ $errgrep -eq 0 ]; then cat <<\EOFs3 >> status.out @@ -3282,11 +3284,11 @@ cat <<\EOFs3 >> status.out EOFs3 - grep -e " SAT. ID " -e " %#" $pgmout | sed "s/%#/ /g" \ + grep --text -e " SAT. ID " -e " %#" $pgmout | sed "s/%#/ /g" \ >> status.out fi - grep -q -e " replicated observations" -e " #%" $pgmout + grep --text -q -e " replicated observations" -e " #%" $pgmout errgrep=$? if [ $errgrep -eq 0 ]; then cat <<\EOFs4 >> status.out @@ -3298,7 +3300,7 @@ cat <<\EOFs4 >> status.out EOFs4 - grep -e " replicated observations" -e " #%" $pgmout | \ + grep --text -e " replicated observations" -e " #%" $pgmout | \ sed "s/#%/ /g" >> status.out fi diff --git a/ush/dumpjb b/ush/dumpjb index 9a2c47f..9e53860 100755 --- a/ush/dumpjb +++ b/ush/dumpjb @@ -476,7 +476,7 @@ # bufr_dupsst bufr_dupmrg bufr_edtbfr bufr_quipc # bufr_chkbfr bufr_combfr bufr_dupair bufr_dupshp # bufr_dupmar bufr_geofil bufr_duprad bufr_dupsyp -# bufr_duptac bufr_dupcman bufr_dupupr +# bufr_duptac bufr_dupcman bufr_dupupr bufr_dupsno # # Remarks: # @@ -584,6 +584,7 @@ # SHPX - path to DUPSHP executable (default:$DUEX/bufr_dupshp) # CMANX - path to DUPCMAN executable (default:$DUEX/bufr_dupcman) # SYPX - path to DUPSYP executable (default:$DUEX/bufr_dupsyp) +# SNOX - path to DUPSNO executable (default:$DUEX/bufr_dupsno) # TACX - path to DUPTAC executable (default:$DUEX/bufr_duptac) # UPRX - path to DUPUPR executable (default:$DUEX/bufr_dupupr) # EDTX - path to EDTBFR executable (default:$DUEX/bufr_edtbfr) @@ -1004,6 +1005,7 @@ echo [ -z "$SHPX" ] && SHPX=$DUEX/bufr_dupshp [ -z "$CMANX" ] && CMANX=$DUEX/bufr_dupcman [ -z "$SYPX" ] && SYPX=$DUEX/bufr_dupsyp +[ -z "$SNOX" ] && SNOX=$DUEX/bufr_dupsno [ -z "$TACX" ] && TACX=$DUEX/bufr_duptac [ -z "$UPRX" ] && UPRX=$DUEX/bufr_dupupr [ -z "$EDTX" ] && EDTX=$DUEX/bufr_edtbfr @@ -1184,6 +1186,7 @@ AIRX = $AIRX SHPX = $SHPX CMANX = $CMANX SYPX = $SYPX +SNOX = $SNOX TACX = $TACX UPRX = $UPRX EDTX = $EDTX @@ -2043,7 +2046,7 @@ dupsyp_run=no grep -q -e "000.000" -e "000.001" -e "000.002" -e "000.100" -e "000.101" \ -e "000.102" $FILES errgreps=$? -if [ $errgreps -eq 0 ]; then +if [[ $errgreps -eq 0 && $NAME = "adpsfc" ]]; then if [ $DUPC = on ]; then echo "--------------------------------------------------------------------" echo "Duplicate check combined synoptic reports in \ @@ -2068,6 +2071,36 @@ NOT PERFORMED (turned off by user)" fi echo "--------------------------------------------------------------------" fi +rc_dupsno=0 +dupsno_run=no +grep -q -e "000.000" -e "000.001" -e "000.002" -e "000.100" -e "000.101" \ +-e "000.102" $FILES +errgreps=$? +if [[ $errgreps -eq 0 && $NAME = "sfcsno" ]]; then + if [ $DUPC = on ]; then + echo "--------------------------------------------------------------------" + echo "Duplicate check combined synoptic reports in \ +000.000 - 000.002, 000.100 - 000.102" + echo "Duplicate check program: $SNOX" + echo "--------------------------------------------------------------------" + echo $FILES + cat $FILES + time cat $FILES | $SNOX + if [ ! -s fort.60 ]; then + rc_dupsno=99 + else + dupsno_run=yes + fi + rm fort.60 2>/dev/null + else + echo "--------------------------------------------------------------------" + echo "Duplicate check of combined synoptic reports in \ +000.000 - 000.002, 000.100 - 000.102 \ +NOT PERFORMED (turned off by user)" + echo "--------------------------------------------------------------------" + fi + echo "--------------------------------------------------------------------" +fi rc_dupshp=0 dupshp_run=no grep -q -e "001.001" -e "001.013" -e "001.101" -e "001.113" $FILES @@ -2205,6 +2238,11 @@ if [ $dupsyp_run = yes ]; then echo "Duplicate checking completed for combined synoptic \ reports in 000.000 - 000.002, 000.100 - 000.102" fi +if [ $dupsno_run = yes ]; then + echo + echo "Duplicate checking completed for combined synoptic \ +reports in 000.000 - 000.002, 000.100 - 000.102" +fi if [ $dupshp_run = yes ]; then echo echo "Duplicate checking completed for combined ship reports in 001.001 \ @@ -2332,7 +2370,7 @@ NFIN=`ls *.fin 2>/dev/null|wc -w` [ $NFIN -lt $NOUT ] && RETG=99 [ $rc_dupair -eq 99 -o $rc_dupshp -eq 99 -o $rc_dupsyp -eq 99 -o \ $rc_duptac -eq 99 -o $rc_edtbfr -eq 99 -o $rc_quipc -eq 99 -o \ - $rc_chkbfr -eq 99 -o $rc_dupcman -eq 99 ] && RETG=99 + $rc_chkbfr -eq 99 -o $rc_dupcman -eq 99 -o $rc_dupsno -eq 99 ] && RETG=99 ###########if [ $RETG -ge 22 ] if [ $RETG -gt 22 ]; then diff --git a/versions/VERSION b/versions/VERSION index 26aaba0..f0bb29e 100644 --- a/versions/VERSION +++ b/versions/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0