[ADC] Configure certificates (#2815)

* Update yum deps for ADC

* Fix cloudwatch key url domain for ADC

* [ADC] Set default node package for build image command

* [ADC] Configure certificates for compute fleet manager

* [ADC Build Image] Source AWS_CA_BUNDLE during build

* [ADC] Configure certificates for login nodes in ADC

Author: hgreebe

cookbooks/aws-parallelcluster-awsbatch/recipes/install.rb

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 return if platform?('redhat')
+return if aws_region.start_with?("us-iso")
 
 include_recipe "::awsbatch_virtualenv"
 
@@ -28,9 +29,6 @@
 
 # Check whether install a custom aws-parallelcluster-awsbatch-cli package or the standard one
 # Install awsbatch cli into awsbatch virtual env
-if aws_region.start_with?("us-iso") && !node['cluster']['custom_awsbatchcli_package'].empty?
-  node.default['cluster']['custom_awsbatchcli_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/awsbatch/aws-parallelcluster.tgz"
-end
 if !node['cluster']['custom_awsbatchcli_package'].nil? && !node['cluster']['custom_awsbatchcli_package'].empty?
   # Install custom aws-parallelcluster package
   bash "install aws-parallelcluster-awsbatch-cli" do
@@ -45,7 +43,7 @@
       curl --retry 3 -L -o aws-parallelcluster.tgz ${custom_package_url}
       mkdir aws-parallelcluster-awsbatch-cli
       tar -xzf aws-parallelcluster.tgz --directory aws-parallelcluster-awsbatch-cli
-      cd aws-parallelcluster-awsbatch-cli/*aws-parallelcluster-*
+      cd aws-parallelcluster-awsbatch-cli/*aws-parallelcluster*
 
       #{node['cluster']['awsbatch_virtualenv_path']}/bin/pip install awsbatch-cli/
     CLI

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -56,7 +56,7 @@
     rm -fr aws-parallelcluster-custom-node
     mkdir aws-parallelcluster-custom-node
     tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node
-    cd aws-parallelcluster-custom-node/*aws-parallelcluster-node-*
+    cd aws-parallelcluster-custom-node/*aws-parallelcluster-node*
     pip install .
     deactivate
   NODE

cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb

@@ -34,7 +34,9 @@
 end
 
 if aws_region.start_with?("us-iso") && !is_custom_node?
-  node.default['cluster']['custom_node_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/node/aws-parallelcluster-node.tgz"
+  node_package = "aws-parallelcluster-node-#{node['cluster']['parallelcluster-node-version']}.tgz"
+
+  node.default['cluster']['custom_node_package'] = "#{node['cluster']['s3_url']}/parallelcluster/#{node['cluster']['parallelcluster-node-version']}/node/#{node_package}"
 end
 
 if is_custom_node?

cookbooks/aws-parallelcluster-environment/resources/cloudwatch/partial/_cloudwatch_common.rb

@@ -27,9 +27,16 @@
 
   action_cloudwatch_prerequisite
 
+  region = node['cluster']['region']
+  key_path = "amazoncloudwatch-agent/assets/amazon-cloudwatch-agent.gpg"
+  cloudwatch_key_url = "https://s3.amazonaws.com/#{key_path}"
+  if region.start_with?("us-iso")
+    cloudwatch_key_url = "https://s3.#{aws_region}.#{aws_domain}/#{key_path}"
+  end
+
   public_key_local_path = "#{node['cluster']['sources_dir']}/amazon-cloudwatch-agent.gpg"
   remote_file public_key_local_path do
-    source 'https://s3.amazonaws.com/amazoncloudwatch-agent/assets/amazon-cloudwatch-agent.gpg'
+    source cloudwatch_key_url
     retries 3
     retry_delay 5
     action :create_if_missing

cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh

@@ -13,4 +13,12 @@ echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profi
 
 echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh
 
-echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep
+echo "export REQUESTS_CA_BUNDLE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh
+
+echo "export SSL_CERT_FILE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh
+
+echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE REQUESTS_CA_BUNDLE SSL_CERT_FILE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep
+
+source /etc/profile.d/aws-cli-default-config.sh
+
+sudo aws configure set ca_bundle "$CA_BUNDLE"

cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_amazon2.rb

@@ -49,7 +49,7 @@ def default_packages
 
   if aws_region.start_with?("us-iso")
     remote_file "epel_deps.tar.gz" do
-      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel7/#{node['kernel']['machine']}/epel_deps.tar.gz"
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel7/#{node['kernel']['machine']}/deps-v2.tar.gz"
       mode '0644'
       retries 3
       retry_delay 5

cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb

@@ -43,7 +43,7 @@ def default_packages
 action :install_extras do
   if aws_region.start_with?("us-iso")
     remote_file "epel_deps.tar.gz" do
-      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel8/x86_64/epel_deps.tar.gz"
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel8/x86_64/deps-v2.tar.gz"
       mode '0644'
       retries 3
       retry_delay 5

cookbooks/aws-parallelcluster-shared/attributes/environment.rb

@@ -5,4 +5,5 @@
 
 # URL for ParallelCluster Artifacts stored in public S3 buckets
 # ['cluster']['region'] will need to be defined by image_dna.json during AMI build.
-default['cluster']['artifacts_s3_url'] = "https://#{node['cluster']['region']}-aws-parallelcluster.s3.#{node['cluster']['region']}.#{node['cluster']['aws_domain']}/archives"
+default['cluster']['s3_url'] = "https://#{node['cluster']['region']}-aws-parallelcluster.s3.#{node['cluster']['region']}.#{node['cluster']['aws_domain']}"
+default['cluster']['artifacts_s3_url'] = "#{node['cluster']['s3_url']}/archives"

cookbooks/aws-parallelcluster-slurm/templates/default/slurm/fleet_status_manager_program.erb

@@ -1,2 +1,3 @@
 #!/bin/bash
+source /etc/profile.d/aws-cli-default-config.sh
 sudo -u <%= node['cluster']['cluster_admin_user'] %> <%= node_virtualenv_path %>/bin/slurm_fleet_status_manager "$@"

cookbooks/aws-parallelcluster-slurm/templates/default/slurm/resume_program.erb

@@ -2,6 +2,8 @@
 # ResumeProgram should read SLURM_RESUME_FILE within ten seconds of starting to guarantee that it still exists.
 # ref https://slurm.schedmd.com/power_save.html#tolerance
 
+source /etc/profile.d/aws-cli-default-config.sh
+
 trap "rm -f ${SLURM_RESUME_FILE_TMP}" EXIT
 
 SLURM_RESUME_FILE_TMP=$(mktemp)