Updating with new Jenkinsfile.

ci/Jenkinsfile4AWS

@@ -4,10 +4,12 @@ def CUSTOM_WORKSPACE = 'none'
 def HOMEgfs = 'none'
 def CI_CASES = ''
 def GH = 'none'
+//Trivial change
 // Location of the custom workspaces for each machine in the CI system.  They are persistent for each iteration of the PR.
-def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins']
-def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
+def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'Awsepicglobalworkflow']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins']
+def repo_url = 'https://github.com/NOAA-EMC/global-workflow.git'
+def aws_gw_name = 'global-workflow'
 def STATUS = 'Passed'
 
 pipeline {
@@ -44,7 +46,9 @@ pipeline {
                         for (label in pullRequest.labels) {
                             if (label.matches("CI-(.*?)-Ready")) {
                                 def machine_name = label.split('-')[1].toString().toLowerCase()
+                                print machine_name
                                 jenkins.model.Jenkins.get().computers.each { c ->
+                                    print c.node.selfLabel.name
                                     if (c.node.selfLabel.name == NodeName[machine_name]) {
                                         run_nodes.add(c.node.selfLabel.name)
                                     }
@@ -56,7 +60,7 @@ pipeline {
                             run_nodes.init().each { node ->
                                 def machine_name = node.split('-')[0].toLowerCase()
                                 echo "Spawning job on node: ${node} with machine name: ${machine_name}"
-                                build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [
+                                build job: "/${aws_gw_name}/EPIC-AWS-CI-Pipeline/PR-${env.CHANGE_ID}", parameters: [
                                     string(name: 'machine', value: machine_name),
                                     string(name: 'Node', value: node) ],
                                     wait: false
@@ -79,11 +83,11 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea', 'awsepicglobalworkflow'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea', 'Awsepicglobalworkflow'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
-                        CUSTOM_WORKSPACE = "${WORKSPACE}"
-                        HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
-                        sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/global-workflow; mkdir -p ${CUSTOM_WORKSPACE}/global-workflow")
+                        CUSTOM_WORKSPACE = "${custom_workspace[machine]}/${env.CHANGE_ID}/${aws_gw_name}"
+                        HOMEgfs = "${CUSTOM_WORKSPACE}/${aws_gw_name}"
+                        sh(script: "rm -Rf ${CUSTOM_WORKSPACE}; mkdir -p ${CUSTOM_WORKSPACE}/${aws_gw_name}")
                         sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS; mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
                     }
@@ -95,17 +99,23 @@ pipeline {
         stage('3. Build System') {
             agent { label NodeName[machine].toLowerCase() }
             steps {
+                dir("${HOMEgfs}") {
+                checkout scm
+                }
                 catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                     script {
                         ws(HOMEgfs) {
                             echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
                             try {
-                                checkout scm
+                                //sh(script: "git clone --recursive ${repo_url}")
+                                sh(script: "ls ${HOMEgfs}")
                             } catch (Exception e) {
                                 echo "Failed to checkout the code on ${Machine} using scm in ${HOMEgfs}, try again ..."
                                 sleep time: 45, unit: 'SECONDS'
                                 try {
-                                    checkout scm
+                                    // checkout scm
+                                    //sh(script: "git clone --recursive ${repo_url}")
+                                    sh(script: "ls ${HOMEgfs}/sorc")
                                 } catch (Exception ee) {
                                     echo "Failed to checkout the code on ${Machine} using scm in ${HOMEgfs}: ${ee.getMessage()}"
                                     if (env.CHANGE_ID) {
@@ -120,7 +130,9 @@ pipeline {
                             def error_logs_message = ""
                             dir("${HOMEgfs}/sorc") {
                                 try {
-                                    sh(script: './build_compute.sh all')  // build the global-workflow executables
+                                    sh(script: 'ls ./build_compute.sh')	// list files here to make sure all files exist.
+                                    sh(script: './build_compute.sh gfs')  // build the global-workflow executables
+                                    sh(script: './build_compute.sh gefs')  // build the global-workflow executables
                                 } catch (Exception error_build) {
                                     echo "Failed to build global-workflow: ${error_build.getMessage()}"
                                     if ( fileExists("logs/error.logs") ) {

ci/cases/pr/C48_S2SW.yaml

@@ -12,3 +12,4 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+

ci/cases/pr/C48_S2SW_extended.yaml

@@ -19,4 +19,5 @@ skip_ci_on_hosts:
   - gaeac6
   - orion
   - hercules
+  - awsepicglobalworkflow
   - wcoss2  # TODO run on WCOSS2 once the gfs_waveawipsbulls job is fixed

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -21,3 +21,4 @@ skip_ci_on_hosts:
   - gaeac6
   - gaeac5
   - orion
+  - awsepicglobalworkflow

ci/cases/pr/C48mx500_hybAOWCDA.yaml

@@ -22,3 +22,4 @@ skip_ci_on_hosts:
   - gaeac5
   - gaeac6
   - orion
+  - awsepicglobalworkflow

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -17,3 +17,6 @@ arguments:
   interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - awsepicglobalworkflow

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -22,3 +22,4 @@ skip_ci_on_hosts:
   - gaeac5
   - gaeac6
   - hercules
+  - awsepicglobalworkflow

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -22,3 +22,4 @@ skip_ci_on_hosts:
   - gaeac6
   - orion
   - hercules
+  - awsepicglobalworkflow

ci/cases/pr/C96_atm3DVar.yaml

@@ -18,3 +18,4 @@ arguments:
 
 skip_ci_on_hosts:
   - wcoss2
+  - awsepicglobalworkflow

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -22,3 +22,4 @@ skip_ci_on_hosts:
   - gaeac6
   - orion
   - hercules
+  - awsepicglobalworkflow

ci/cases/pr/C96mx100_S2S.yaml

@@ -20,3 +20,4 @@ arguments:
 skip_ci_on_hosts:
   - gaeac6
   - gaeac5
+  - awsepicglobalworkflow

ci/scripts/utils/ci_utils.sh

@@ -119,11 +119,20 @@ function create_experiment () {
   case=$(basename "${yaml_config}" .yaml) || true
   export pslot=${case}_${pr_sha}
 
-  source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+  if [[ ${MACHINE_ID} == "noaacloud" ]]; then
+      source "${HOMEgfs}/ci/platforms/config.${PW_CSP}"
+  else
+      source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+  fi
+
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
   # Remove RUNDIRS dir incase this is a retry (STMP now in host file)
-  STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s)
+  if [[ ${MACHINE_ID} == "noaacloud" ]]; then
+      STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s)
+  else
+      STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s)
+  fi
   echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry"
   rm -Rf "${STMP}/RUNDIRS/${pslot}"
 

ci/scripts/utils/launch_java_agent.sh

@@ -76,6 +76,8 @@ source "${HOMEGFS_}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 )
     echo "Launch Jenkins Java Controler on ${MACHINE_ID}";;
+  noaacloud )
+    echo "Launch Jenkins Java Controler on ${PW_CSP}";;
   *)
     echo "Unsupported platform. Exiting with error."
     exit 1;;
@@ -87,7 +89,12 @@ rm -f "${LOG}"
 HOMEgfs="${HOMEGFS_}" source "${HOMEGFS_}/ush/module-setup.sh"
 module use "${HOMEGFS_}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
-source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"
+
+if [[ ${MACHINE_ID} == "noaacloud" ]]; then
+  source "${HOMEgfs_}/ci/platforms/config.${PW_CSP}"
+else
+  source "${HOMEgfs_}/ci/platforms/config.${MACHINE_ID}"
+fi
 
 JAVA_HOME="${JENKINS_AGENT_LAUNCH_DIR}/JAVA/jdk-17.0.10"
 if [[ ! -d "${JAVA_HOME}" ]]; then

parm/config/gfs/config.resources.AWSPW

@@ -4,13 +4,7 @@
 
 export is_exclusive="True"
 unset memory
-
-# shellcheck disable=SC2312
-for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
-  unset "${mem_var}"
-done
-
-step=$1
+unset "memory_${RUN}"
 
 case ${step} in
   "fcst" | "efcs")
@@ -24,6 +18,12 @@ case ${step} in
     ;;
 
 
+  "atmos_products" | "oceanice_products" | "wavepostsbs" )
+    export PARTITION_BATCH="process"
+    max_tasks_per_node=24
+    ;;
+
+
   *)
     export PARTITION_BATCH="process"
     max_tasks_per_node=24