Feature/add acio jumphost

README.md

@@ -7,3 +7,6 @@ packer build srw-cluster.pkr.hcl -var "date=4May2023"
 ### AWS LandDA configuration:
 vi srw-cluster.pkr.hcl #edit line 141: from srw-cluster-start-script.sh to landda-cluster-container-start-script.sh
 packer build srw-cluster.pkr.hcl -var "date=4May2023"
+
+### Required Software
+Version 3.7.1 of AWS ParallelCluster

scripts/deployment/cluster_start_script_v3.sh

@@ -32,3 +32,6 @@ chown -R ubuntu /home/ubuntu/ufs-srweather-app
 
 echo 'Deleting crontab entries'
 crontab -u ubuntu -r
+
+echo 'Installing Bastion Key'
+aws ssm get-parameter --region us-east-1 --name bastion_public_key | jq -r .Parameter.Value >> ~/.ssh/authorized_keys

scripts/deployment/generateClusters.sh

@@ -1,4 +1,4 @@
 for i in $(seq 1 1 1)
     do 
-        pcluster create-cluster --region us-east-1 --cluster-name srwv2-cluster-$i --cluster-configuration srwcluster_nodeconfig_v1.yaml --rollback-on-failure false --debug
-    done
+        pcluster create-cluster --region us-east-1 --cluster-name srwv22-cluster-$i --cluster-configuration srwcluster_nodeconfig_v1.yaml --rollback-on-failure false --debug
+    done

scripts/deployment/software_deployments/image_build_container_ams24.sh

@@ -0,0 +1,90 @@
+###install go###
+wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz
+tar -xvf go1.21.6.linux-amd64.tar.gz
+cd go
+export PATH=$PATH:/home/ubuntu/go/bin
+export GOPATH=/home/ubuntu/go
+export GOBIN=/home/ubuntu/go/bin
+
+###Install singularity###
+cd /home/ubuntu
+wget https://github.com/sylabs/singularity/releases/download/v3.11.0/singularity-ce-3.11.0.tar.gz
+tar -xzf singularity-ce-3.11.0.tar.gz
+cd singularity-ce-3.11.0/
+sudo apt-get install libseccomp-dev
+sudo apt-get update
+sudo apt-get install libglib2.0-dev
+./mconfig &&     make -C ./builddir &&     sudo make -C ./builddir install
+
+###Build the container image###
+cd /home/ubuntu
+sudo singularity build --sandbox ubuntu20.04-intel-srwapp docker://noaaepic/ubuntu20.04-intel-srwapp:release-public-v2.2.0
+
+###Upgrade lmod/Lua###
+cd /home/ubuntu
+sudo apt install lua5.3
+sudo apt remove lua5.2
+wget https://sourceforge.net/projects/lmod/files/Lmod-8.6.tar.bz2
+tar xvfj Lmod-8.6.tar.bz2
+cd Lmod-8.6
+./configure --prefix=/opt/apps
+sudo make install
+source /opt/apps/lmod/lmod/init/bash
+
+###Install ruby and ruby-dev###
+cd /home/ubuntu
+sudo apt-get install ruby
+sudo apt-get install ruby-dev
+
+###Install miniconda###
+cd /home/ubuntu
+git clone -b feature/ufs_srw_public_2.2.0 https://github.com/NOAA-EPIC/miniconda3.git
+cd miniconda3/
+sed -i "s|lustre|home\/ubuntu|g" miniconda3template.lua
+./miniconda3_install.sh /home/ubuntu/miniconda3   4.12.0
+./miniconda3_regional_workflow_env.sh   /home/ubuntu/miniconda3  4.12.0
+./miniconda3_workflow_tools_env.sh   /home/ubuntu/miniconda3  4.12.0
+./miniconda3_regional_workflow_cmaq_env.sh   /home/ubuntu/miniconda3  4.12.0
+# Load the module:
+module use /home/ubuntu/miniconda3/modulefiles
+module load miniconda3/4.12.0
+cd /home/ubuntu/miniconda3/4.12.0/lib/
+mv libtinfo.so.6 libtinfo.so.6_bac
+
+###Install rocoto###
+cd /home/ubuntu
+PREFIX="/home/ubuntu/rocoto"
+mkdir -p $PREFIX && cd $PREFIX
+git clone -b 1.3.6 https://github.com/christopherwharrop/rocoto.git 1.3.6
+cd 1.3.6
+./INSTALL 2>&1 | tee rocoto-1.3.6.install.log
+# Prepare a modulefile for rocoto
+cd $PREFIX
+export ROCOTOBIN=$PREFIX/1.3.6/bin
+export ROCOTOLIB=$PREFIX/1.3.6/lib
+mkdir $PREFIX/modulefiles
+mkdir $PREFIX/modulefiles/rocoto
+touch $PREFIX/modulefiles/rocoto/1.3.6.lua
+cat > modulefiles/rocoto/1.3.6.lua << EOF
+help([[
+  Set environment variables for rocoto workflow manager)
+]])
+
+-- Make sure another version of the same package is not already loaded
+conflict("rocoto")
+
+-- Set environment variables
+prepend_path("PATH","$ROCOTOBIN")
+prepend_path("LD_LIBRARY_PATH","$ROCOTOLIB")
+EOF
+# Verify the module could be loaded:
+module use /$PREFIX/modulefiles
+module load rocoto/1.3.6
+
+###Add needed data###
+cd /scratch
+wget https://noaa-ufs-srw-pds.s3.amazonaws.com/current_srw_release_data/fix_data.tgz
+tar xfz fix_data.tgz
+wget https://noaa-ufs-srw-pds.s3.amazonaws.com/current_srw_release_data/gst_data.tgz
+tar xfz gst_data.tgz
+# After untaring the files, directories ./fix and ./input_model_data

scripts/deployment/srwcluster_nodeconfig_v1.yaml

@@ -3,11 +3,11 @@
 Region: us-east-1
 Image:
   Os: ubuntu2004
-  CustomAmi: ami-030cf29e6d5d8724a #Must modify
+  CustomAmi: ami-08100890884d98d9d #Must modify 
 HeadNode:
   InstanceType: c5.2xlarge
   Networking:
-    SubnetId: ami-030cf29e6d5d8724a #Must modify
+    SubnetId: subnet-078043a467c391dfd #Must modify (public subnet)
   Ssh:
     KeyName: epic_workshop
   LocalStorage:
@@ -18,10 +18,11 @@ HeadNode:
       Throughput: 1000
   Iam:
     AdditionalIamPolicies:
-      - Policy: arn:aws:iam::aws:policy/AmazonS3FullAccess
+      - Policy: arn:aws:iam::aws:policy/AmazonS3FullAccess 
+      - Policy: arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore 
   CustomActions:
     OnNodeConfigured:
-      Script: s3://epic.sandbox.srw/cluster_start_script_v3.sh #Must modify
+      Script: s3://epic-srw-sandbox/cluster_start_script_v3.sh #Must modify
 Scheduling:
   Scheduler: slurm
   SlurmSettings:
@@ -45,13 +46,13 @@ Scheduling:
       MaxCount: 2
     Networking:
       SubnetIds:
-      - subnet-04d911e4b55853ef7 #Must modify
+      - subnet-07bd52a747ba1e525 #Must modify
       AssignPublicIp: true
       PlacementGroup:
         Enabled: true
     CustomActions:
       OnNodeConfigured:
-        Script: s3://epic.sandbox.srw/cluster_start_script_v3.sh #Must modify
+        Script: s3://epic-srw-sandbox/cluster_start_script_v3.sh #Must modify
 SharedStorage:
   - MountDir: /scratch
     Name: ebs

scripts/srw-cluster-start-script.sh

@@ -137,3 +137,94 @@ cd /opt/ufs-srweather-app/build
 cmake -DCMAKE_CXX_COMPILER=mpiicpc -DCMAKE_C_COMPILER=mpiicc -DCMAKE_FC_COMPILER=mpiifort -DCMAKE_INSTALL_PREFIX=.. ..
 make -j 8
 echo "UFS SRW App build completed!"
+###install go###
+echo 'Installing Go'
+wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz
+tar -xvf go1.21.6.linux-amd64.tar.gz
+cd go
+export PATH=$PATH:/home/ubuntu/go/bin
+export GOPATH=/home/ubuntu/go
+export GOBIN=/home/ubuntu/bin
+###Install singularity###
+echo 'Installing Singularity'
+cd /home/ubuntu
+wget https://github.com/sylabs/singularity/releases/download/v3.11.0/singularity-ce-3.11.0.tar.gz
+tar -xzf singularity-ce-3.11.0.tar.gz
+cd singularity-ce-3.11.0/
+sudo apt-get install libseccomp-dev
+sudo apt-get update
+sudo apt-get install libglib2.0-dev
+./mconfig &&     make -C ./builddir &&     sudo make -C ./builddir install
+###Build the container image###
+echo 'Installing Container'
+cd /home/ubuntu
+sudo singularity build --sandbox ubuntu20.04-intel-srwapp docker://noaaepic/ubuntu20.04-intel-srwapp:release-public-v2.2.0
+###Upgrade lmod/Lua###
+echo 'Installing Lmod/Lua'
+cd /home/ubuntu
+sudo apt install lua5.3
+sudo apt remove lua5.2
+wget https://sourceforge.net/projects/lmod/files/Lmod-8.6.tar.bz2
+tar xvfj Lmod-8.6.tar.bz2
+cd Lmod-8.6
+./configure --prefix=/opt/apps
+sudo make install
+source /opt/apps/lmod/lmod/init/bash
+###Install ruby and ruby-dev###
+echo 'Installing Ruby'
+cd /home/ubuntu
+sudo apt-get install ruby
+sudo apt-get install ruby-dev
+###Install miniconda###
+echo 'Installing Miniconda'
+cd /home/ubuntu
+git clone -b feature/ufs_srw_public_2.2.0 https://github.com/NOAA-EPIC/miniconda3.git
+cd miniconda3/
+sed -i "s|lustre|home\/ubuntu|g" miniconda3template.lua
+./miniconda3_install.sh /home/ubuntu/miniconda3   4.12.0
+./miniconda3_regional_workflow_env.sh   /home/ubuntu/miniconda3  4.12.0
+./miniconda3_workflow_tools_env.sh   /home/ubuntu/miniconda3  4.12.0
+./miniconda3_regional_workflow_cmaq_env.sh   /home/ubuntu/miniconda3  4.12.0
+# Load the module:
+module use /home/ubuntu/miniconda3/modulefiles
+module load miniconda3/4.12.0
+cd /home/ubuntu/miniconda3/4.12.0/lib/
+mv libtinfo.so.6 libtinfo.so.6_bac
+###Install rocoto###
+echo 'Installing rocoto'
+cd /home/ubuntu
+PREFIX="/home/ubuntu/rocoto"
+mkdir -p $PREFIX && cd $PREFIX
+git clone -b 1.3.6 https://github.com/christopherwharrop/rocoto.git 1.3.6
+cd 1.3.6
+./INSTALL 2>&1 | tee rocoto-1.3.6.install.log
+# Prepare a modulefile for rocoto
+cd $PREFIX
+export ROCOTOBIN=$PREFIX/1.3.6/bin
+export ROCOTOLIB=$PREFIX/1.3.6/lib
+mkdir $PREFIX/modulefiles
+mkdir $PREFIX/modulefiles/rocoto
+touch $PREFIX/modulefiles/rocoto/1.3.6.lua
+cat > modulefiles/rocoto/1.3.6.lua << EOF
+help([[
+  Set environment variables for rocoto workflow manager)
+]])
+
+-- Make sure another version of the same package is not already loaded
+conflict("rocoto")
+
+-- Set environment variables
+prepend_path("PATH","$ROCOTOBIN")
+prepend_path("LD_LIBRARY_PATH","$ROCOTOLIB")
+EOF
+# Verify the module could be loaded:
+module use /$PREFIX/modulefiles
+module load rocoto/1.3.6
+###Add needed data###
+echo 'Installing Data'
+cd /data
+wget https://noaa-ufs-srw-pds.s3.amazonaws.com/current_srw_release_data/fix_data.tgz
+tar xfz fix_data.tgz
+wget https://noaa-ufs-srw-pds.s3.amazonaws.com/current_srw_release_data/gst_data.tgz
+tar xfz gst_data.tgz
+# After untaring the files, directories ./fix and ./input_model_data

srw-cluster.pkr.hcl

@@ -2,6 +2,10 @@
 # Variables for AWS builders
 ###
 
+locals {
+  now = formatdate("YYYYMMDD-hhmmss", timestamp())
+}
+
 #Add multiple regions: default     = ["us-east-1","us-east-2"]
 variable "aws_ami_regions" {
   description = "List of regions to copy the AMIs to. Tags and attributes are copied along with the AMIs"
@@ -46,9 +50,9 @@ variable "aws_source_ami_filter_ubuntu_2004_hvm" {
     owners = list(string)
   })
   default = {
-    name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
+    name = "aws-parallelcluster-3.7.1-ubuntu-2004-lts-hvm-x86_64-202309151532 2023-09-15T15-36-35.608Z"
     owners = [
-      "099720109477"
+      "247102896272"
     ]
   }
 }
@@ -59,6 +63,7 @@ variable "aws_temporary_security_group_source_cidrs" {
   default     = ["0.0.0.0/0"]
 }
 
+
 ###
 # Variables for Azure builders
 ###
@@ -88,7 +93,7 @@ variable "root_volume_size" {
 ###
 
 source "amazon-ebs" "base" {
-  ami_name                    = "srw-cluster-{{date}}.x86_64-gp3"
+  ami_name                    = "srw-cluster-${local.now}.x86_64-gp3"
   ami_regions                 = var.aws_ami_regions
   ami_users                   = var.aws_ami_users
   ami_groups                  = var.aws_ami_groups
@@ -111,8 +116,8 @@ source "amazon-ebs" "base" {
   ssh_pty                               = true
   ssh_timeout                           = "60m"
   ssh_username                          = var.aws_ssh_username
-  subnet_id                             = "subnet-04bae583ce498ab48"
-  tags                                  = { Name = "SRW-Cluster-{{date}}" }
+  subnet_id                             = "subnet-04d911e4b55853ef7"
+  tags                                  = { Name = "SRW-Cluster-${local.now}" }
   temporary_security_group_source_cidrs = var.aws_temporary_security_group_source_cidrs
 }
 

test.py

@@ -0,0 +1 @@
+import pygrib