R21C: Updates required for ADAS to work on Milan (#360)

* swap gmao w/ that in separate repo

* now using repo as opposed to dir

* changes for milan

* comments on changes

.gitignore

@@ -1,3 +1,6 @@
 /@GEOS_Util/
 /GEOS_Util/
 /GEOS_Util@/
+/@GMAO_pyobs/
+/GMAO_pyobs/
+/GMAO_pyobs@/

CHANGELOG.md

@@ -15,13 +15,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [v1.7.2+R21C_v1.0.3] - 2024-11-07
+
+These are minor changes to GMAO_Shared required for system to work on Milan. They are largely associated w/ python3 related changes and a few perl adjustments. The main changes are:
+
+- replacement of GMAO_pyobs directory w/ GMAO_pyobs repo
+- changes to some python programs 
+- minor changes in GFIO - in support of GMAO_pyobs and GMAO_ods/pyods changes 
+
+Many of these relate to the revised Aerosol-related software.
+
+## [v1.7.2+R21C_v1.0.2] - 2024-11-05
+
+- Generalize handling of obsys.rc for reanalysis when prebufr files are created
+  within the DAS by preQC
+
 ## [v1.7.2+R21C_v1.0.1] - 2024-03-28
 
 ### Changed
 
 - Updated observation resource files for M21C
-- Generalize handling of obsys.rc for reanalysis when prebufr files are created
-  within the DAS by preQC
 
 ## [v1.7.2+R21C_v1.0.0] - 2024-03-01
 

GMAO_etc/Perl_Config.pm

@@ -101,7 +101,7 @@ sub expand {
     #------------------------------------------
     foreach $var (@vars) {
 
-        ($varname = $var) =~ s/[\$\{\}]//g;
+        ($varname = $var) =~ s/[\$[{}]]//g;
         $val_ENV  = $ENV{$varname};
         $val_eval = eval($var);
 

GMAO_etc/Remote_utils.pm

@@ -1891,16 +1891,24 @@ sub rflist {
 # If there is login information, use ftp to check the file.
 
    if ( $remote_machine eq "http" || $remote_machine eq "https" ) {
+      if ( ! defined( $ENV{PERL_WGET} ) ) {
+         print "PERL_WGET not set.  Using default.\n";
+         $WGET = $DEFAULT_WGET;
+      }
+      else {
+         $WGET = $ENV{PERL_WGET};
+      }
+
 # Reconstruct URL without extra : characters.
       $url         = "${remote_machine}:${remote_file}${extra}";
       $dir         = dirname ("$url");
       $regex_raw   = basename ("$url");
       $regex_raw   =~s/\*/\.\*/g;
       $regex       = "\^${regex_raw}\$";
       print "Using regex=$regex\n";
-      print "$DEFAULT_WGET -O - $dir\n";
+      print "$WGET -O - $dir\n";
 
-      $string=`$DEFAULT_WGET -O - $dir | grep -o \'<a href=[\'\"\'\"\'\"][^\"\'\"\'\"\']*[\'\"\'\"\'\"]\' |  sed -e \'s/\^<a href=[\"\'\"\'\"\']//\' -e \'s/\[\"\'\"\'\"\'\]\$//\' | grep -v -e \'\?\' -e \'\/\' | grep -E $regex | uniq `;
+      $string=`$WGET -O - $dir | grep -o \'<a href=[\'\"\'\"\'\"][^\"\'\"\'\"\']*[\'\"\'\"\'\"]\' |  sed -e \'s/\^<a href=[\"\'\"\'\"\']//\' -e \'s/\[\"\'\"\'\"\'\]\$//\' | grep -v -e \'\?\' -e \'\/\' | grep -E $regex | uniq `;
       print "POPALI string = $string\n";
       @retlist = split(/\n/,$string);
    }
@@ -2333,6 +2341,13 @@ my ( $LOCAL, $local_host_name, $local_domain_name,
 # If there is login information, use ftp to transfer the file.
 
         if ( $remote_machine eq "http" || $remote_machine eq "https" ) {
+           if ( ! defined( $ENV{PERL_WGET} ) ) {
+              print "PERL_WGET not set.  Using default.\n";
+              $WGET = $DEFAULT_WGET;
+           } 
+           else {
+              $WGET=$ENV{PERL_WGET};
+           }
            # $opt_filename handles the case where the URL passed in has two ':' symbols, one after http
            # and the other between the machine name and the file path.
            if ( $opt_filename ) {
@@ -2341,8 +2356,8 @@ my ( $LOCAL, $local_host_name, $local_domain_name,
            else {
              $target = "${remote_machine}:${remote_file}";
            }
-           print "$DEFAULT_WGET -O ${local_file} ${target}\n";
-           `$DEFAULT_WGET -O ${local_file} ${target}`;
+           print "$WGET -O ${local_file} ${target}\n";
+           `$WGET -O ${local_file} ${target}`;
            if ( $? == 1 ) {
               $retcode = 0;
            }

GMAO_etc/WriteLog.pm

@@ -624,7 +624,7 @@ sub expand_EnvVars {
 
     # look for ${var} format
     #-----------------------
-    while ($string =~ m/(\${(\w+)})/)   {
+    while ($string =~ m/(\$\{(\w+)\})/)   {
         $var = $1; $name = $2;
         $var =~ s/\$/\\\$/;
         $string =~ s/$var/$ENV{$name}/;

GMAO_etc/dateutils.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 import datetime, calendar
 
 """
@@ -20,7 +21,7 @@ def dateto_hrs_since_day1CE(curdate,mixedcal=True):
     if mixedcal:
         if curdate < gregstart:
             msg = 'date must be after start of gregorian calendar (15821015)!'
-            raise ValueError, msg
+            raise ValueError( msg )
         difftime = curdate-gregstart
         hrsdiff = 24*difftime.days + difftime.seconds/3600
         return hrsdiff+hrsgregstart
@@ -32,7 +33,7 @@ def hrs_since_day1CE_todate(hrs,mixedcal=True):
     """return datetime.datetime instance given hours since 1-Jan-0001"""
     if hrs < 0.0:
         msg = "hrs must be positive!"
-        raise ValueError, msg
+        raise ValueError( msg )
     delta = datetime.timedelta(hours=1)
     if mixedcal:
         hrs_sincegreg = hrs - hrsgregstart
@@ -124,9 +125,9 @@ def dayofyear(yyyy,mm,dd):
 def getyrmon(day_of_year,yyyy=2001):
     d1 = datetime.datetime(yyyy,1,1)
     if calendar.isleap(d1.year) and day_of_year > 366:
-        raise ValueError, 'not that many days in the year'
+        raise ValueError( 'not that many days in the year' )
     if not calendar.isleap(d1.year) and day_of_year > 365:
-        raise ValueError, 'not that many days in the year'
+        raise ValueError( 'not that many days in the year' )
     d2 = d1 + (day_of_year-1)*datetime.timedelta(days=1)
     return d2.month,d2.day
 
@@ -137,8 +138,8 @@ def daysinmonth(yyyy,mm):
     return calendar.monthrange(yyyy,mm)[1]
 
 if __name__ == "__main__":
-    print dayofyear(2000,2,29)
-    print daysinmonth(2000,2)
-    print datetohrs('0001010100',mixedcal=False)
-    print datetohrs('2001010100',mixedcal=False)
-    print datetohrs('2001010100',mixedcal=True)
+    print (dayofyear(2000,2,29))
+    print (daysinmonth(2000,2))
+    print (datetohrs('0001010100',mixedcal=False))
+    print (datetohrs('2001010100',mixedcal=False))
+    print (datetohrs('2001010100',mixedcal=True))

GMAO_etc/esma_mpirun

@@ -6,7 +6,7 @@
 #           other flags required for the operating environment.
 #
 # note -
-# 1. Any flag which has not been identified as potentially needing 
+# 1. Any flag which has not been identified as potentially needing
 #    to be filtered is passed directly through to the mpirun command.
 # 2. Specifically, the -h (or -help) flag is passed through to the
 #    mpirun command.
@@ -100,7 +100,7 @@ sub get_mpi_type {
 
     # check MPI specific env variable
     #--------------------------------
-    if    ( $ENV{"MVAPICH"}    )             { $mpi_type = "mvapich2" }
+    if    ( $ENV{"MVAPICH"}    )             { $mpi_type = "mvapich"  }
     elsif ( $ENV{"I_MPI_ROOT"} )             { $mpi_type = "intel"    }
     elsif ( $ENV{"OPENMPI"}    )             { $mpi_type = "openmpi"  }
     elsif ( $ENV{"MPICH"}      )             { $mpi_type = "mpich"    }
@@ -110,7 +110,7 @@ sub get_mpi_type {
     # check $MPIHOME env variable
     #----------------------------
     elsif ( $MPIHOME = $ENV{"MPIHOME"} ) {
-        if    ( $MPIHOME =~ m[mvapich2] )    { $mpi_type = "mvapich2" }
+        if    ( $MPIHOME =~ m[mvapich] )     { $mpi_type = "mvapich"  }
         elsif ( $MPIHOME =~ m[/intel/mpi/] ) { $mpi_type = "intel"    }
         elsif ( $MPIHOME =~ m[openmpi] )     { $mpi_type = "openmpi"  }
         elsif ( $MPIHOME =~ m[mpich] )       { $mpi_type = "mpich"    }
@@ -119,7 +119,7 @@ sub get_mpi_type {
     # check $MPI_HOME env variable
     #-----------------------------
     elsif ( $MPIHOME = $ENV{"MPI_HOME"} ) {
-        if    ( $MPIHOME =~ m[mvapich2] )    { $mpi_type = "mvapich2" }
+        if    ( $MPIHOME =~ m[mvapich] )     { $mpi_type = "mvapich"  }
         elsif ( $MPIHOME =~ m[/intel/mpi/] ) { $mpi_type = "intel"    }
         elsif ( $MPIHOME =~ m[openmpi] )     { $mpi_type = "openmpi"  }
         elsif ( $MPIHOME =~ m[mpich] )       { $mpi_type = "mpich"    }
@@ -128,7 +128,7 @@ sub get_mpi_type {
     # check $MPI_ROOT env variable
     #-----------------------------
     elsif ( $MPIHOME = $ENV{"MPI_ROOT"} ) {
-        if    ( $MPIHOME =~ m[mvapich2] )    { $mpi_type = "mvapich2" }
+        if    ( $MPIHOME =~ m[mvapich] )     { $mpi_type = "mvapich"  }
         elsif ( $MPIHOME =~ m[/intel/mpi/] ) { $mpi_type = "intel"    }
         elsif ( $MPIHOME =~ m[/sgi/mpi/] )   { $mpi_type = "mpt"      }
         elsif ( $MPIHOME =~ m[/hpe/mpt/] )   { $mpi_type = "mpt"      }
@@ -208,19 +208,17 @@ sub which_mpi_cmd {
        # NOTE: If you are running mpiexec_mpt at NCCS under SLURM interactive
        #       with MPT, you will *NOT* get the same behavior as under
        #       a batch job. To get the same behaviour, you must run with:
-       #          
+       #
        #          setenv SLURM_DISTRIBUTION=block
        #
        #       This is currently done with both g5das.j and gcm_run.j
-       
+
        chomp($mpicmd = `which mpiexec_mpt`);
     } elsif ($mpi_type eq 'mpich') {
        chomp($mpicmd = `which mpiexec`);
-    } elsif ($mpi_type eq 'mvapich2') {
-       # For now use mpirun_rsh. At NCCS this should be
-       # srun -n but as -np is *external* to esma_mpirun
-       # this requires wholesale script changes
-       chomp($mpicmd = `which mpiexec.mpirun_rsh`);
+    } elsif ($mpi_type eq 'mvapich') {
+       # On SLES 15, mvapich seems to work best with mpiexec
+       chomp($mpicmd = `which mpiexec`);
     } else {
        # Used by both Intel MPI and Open MPI
        chomp($mpicmd = `which mpirun`);
@@ -249,10 +247,6 @@ sub get_xtraflags {
     $xtraflags = "";
     $siteID = get_siteID();
 
-    # flags needed for mvapich2 mpi
-    #------------------------------
-    $xtraflags .= "-export" if $mpi_type eq "mvapich2";
-
     # flags needed for intel mpi
     #---------------------------
     if ($mpi_type eq "intel") {
@@ -343,6 +337,11 @@ sub get_xtraflags {
 
     }
 
+    if ($mpi_type eq "mvapich") {
+       if ($perhost) {
+          $xtraflags .= " -ppn $perhost";
+       }
+    }
 }
 
 #============================================================================

GMAO_etc/nmlread.py

@@ -11,8 +11,8 @@
 
 def my_usage():
     print 
-    print "Usage: ";
-    print "nmlread.py  rcfile namelist variable";
+    print ("Usage: ");
+    print ("nmlread.py  rcfile namelist variable");
     print 
     exit(1)
 
@@ -27,6 +27,6 @@ def my_usage():
 parser = Parser()
 nml = parser.read(rcfile_)
 var=nml[nmlname_][nmlarg_]
-print var
+print (var);
 
 

GMAO_etc/parallel-untar.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 # From: https://github.com/parallel-fs-utils/multi-thread-posix/blob/master/parallel-untar.py

GMAO_etc/pesto

@@ -198,7 +198,7 @@ EOD
 # Process list of URLs
 # --------------------  
   foreach $url ( @url_list ){
-       ( $url = $url ) =~ ( s/\${PESTOROOT}/$pestoroot\//g ); 
+       ( $url = $url ) =~ ( s/\$\{PESTOROOT\}/$pestoroot\//g ); 
        ( $ruser, $rhost, $file_path ) = splitrfile ( $url );
        print "($myname):Processing $url\n";
        $file = basename($file_path);