Change enroot and pyxis urls to point to an s3 bucket (#2806)

* Fix enroot kitchen tests

* Change enroot download url to s3 url

* Change pyxis download url to s3 url

* Fix enroot+caps url

Author: hgreebe

cookbooks/aws-parallelcluster-platform/attributes/platform.rb

@@ -11,7 +11,7 @@
 
 # Enroot + Pyxis
 default['cluster']['enroot']['version'] = '3.4.1'
-default['cluster']['pyxis']['version'] = '0.19.0'
+default['cluster']['pyxis']['version'] = '0.20.0'
 
 # NVidia
 default['cluster']['nvidia']['enabled'] = 'no'

cookbooks/aws-parallelcluster-platform/files/enroot/enroot.template.conf

@@ -0,0 +1,71 @@
+#ENROOT_LIBRARY_PATH        /usr/lib/enroot
+#ENROOT_SYSCONF_PATH        /etc/enroot
+ENROOT_RUNTIME_PATH        /tmp/enroot/user-$(id -u)
+ENROOT_CONFIG_PATH         ${ENROOT_CONFIG_PATH}
+ENROOT_CACHE_PATH          ${ENROOT_CACHE_PATH}
+ENROOT_DATA_PATH           /tmp/enroot/data/user-$(id -u)
+#ENROOT_TEMP_PATH           ${TMPDIR:-/tmp}
+
+# Gzip program used to uncompress digest layers.
+#ENROOT_GZIP_PROGRAM        gzip
+
+# Options passed to zstd to compress digest layers.
+#ENROOT_ZSTD_OPTIONS        -1
+
+# Options passed to mksquashfs to produce container images.
+ENROOT_SQUASH_OPTIONS -noI -noD -noF -noX -no-duplicates
+
+# Make the container root filesystem writable by default.
+ENROOT_ROOTFS_WRITABLE     yes
+
+# Remap the current user to root inside containers by default.
+#ENROOT_REMAP_ROOT          no
+
+# Maximum number of processors to use for parallel tasks (0 means unlimited).
+#ENROOT_MAX_PROCESSORS      $(nproc)
+
+# Maximum number of concurrent connections (0 means unlimited).
+#ENROOT_MAX_CONNECTIONS     10
+
+# Maximum time in seconds to wait for connections establishment (0 means unlimited).
+#ENROOT_CONNECT_TIMEOUT     30
+
+# Maximum time in seconds to wait for network operations to complete (0 means unlimited).
+#ENROOT_TRANSFER_TIMEOUT    0
+
+# Number of times network operations should be retried.
+#ENROOT_TRANSFER_RETRIES    0
+
+# Use a login shell to run the container initialization.
+#ENROOT_LOGIN_SHELL         yes
+
+# Allow root to retain his superuser privileges inside containers.
+#ENROOT_ALLOW_SUPERUSER     no
+
+# Use HTTP for outgoing requests instead of HTTPS (UNSECURE!).
+#ENROOT_ALLOW_HTTP          no
+
+# Include user-specific configuration inside bundles by default.
+#ENROOT_BUNDLE_ALL          no
+
+# Generate an embedded checksum inside bundles by default.
+#ENROOT_BUNDLE_CHECKSUM     no
+
+# Mount the current user's home directory by default.
+ENROOT_MOUNT_HOME          no
+
+# Restrict /dev inside the container to a minimal set of devices.
+ENROOT_RESTRICT_DEV        no
+
+# Always use --force on command invocations.
+#ENROOT_FORCE_OVERRIDE      no
+
+# SSL certificates settings:
+#SSL_CERT_DIR
+#SSL_CERT_FILE
+
+# Proxy settings:
+#all_proxy
+#no_proxy
+#http_proxy
+#https_proxy

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_common.rb

@@ -24,14 +24,22 @@
   return if on_docker?
   return unless enroot_installed
 
+  cookbook_file "/tmp/enroot.template.conf" do
+    source 'enroot/enroot.template.conf'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0755'
+    action :create_if_missing
+  end
+
   bash "Configure enroot" do
     user 'root'
     code <<-ENROOT_CONFIGURE
       set -e
       ENROOT_CONFIG_RELEASE=pyxis
       SHARED_DIR=#{node['cluster']['shared_dir']}
       NONROOT_USER=#{node['cluster']['cluster_user']}
-      wget -O /tmp/enroot.template.conf https://raw.githubusercontent.com/aws-samples/aws-parallelcluster-post-install-scripts/${ENROOT_CONFIG_RELEASE}/pyxis/enroot.template.conf
       mkdir -p ${SHARED_DIR}/enroot
       chown ${NONROOT_USER} ${SHARED_DIR}/enroot
       ENROOT_CACHE_PATH=${SHARED_DIR}/enroot envsubst < /tmp/enroot.template.conf > /tmp/enroot.conf

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_debian.rb

@@ -37,11 +37,11 @@
 end
 
 def enroot_url
-  "https://github.com/NVIDIA/enroot/releases/download/v#{package_version}/enroot_#{package_version}-1_#{arch_suffix}.deb"
+  "#{node['cluster']['artifacts_s3_url']}/dependencies/enroot/enroot_#{package_version}-1_#{arch_suffix}.deb"
 end
 
 def enroot_caps_url
-  "https://github.com/NVIDIA/enroot/releases/download/v#{package_version}/enroot+caps_#{package_version}-1_#{arch_suffix}.deb"
+  "#{node['cluster']['artifacts_s3_url']}/dependencies/enroot/enroot%2Bcaps_#{package_version}-1_#{arch_suffix}.deb"
 end
 
 def arch_suffix

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_rhel.rb

@@ -34,11 +34,11 @@
 end
 
 def enroot_url
-  "https://github.com/NVIDIA/enroot/releases/download/v#{package_version}/enroot-#{package_version}-1.el8.#{arch_suffix}.rpm"
+  "#{node['cluster']['artifacts_s3_url']}/dependencies/enroot/enroot-#{package_version}-1.el8.#{arch_suffix}.rpm"
 end
 
 def enroot_caps_url
-  "https://github.com/NVIDIA/enroot/releases/download/v#{package_version}/enroot+caps-#{package_version}-1.el8.#{arch_suffix}.rpm"
+  "#{node['cluster']['artifacts_s3_url']}/dependencies/enroot/enroot+caps-#{package_version}-1.el8.#{arch_suffix}.rpm"
 end
 
 def arch_suffix

cookbooks/aws-parallelcluster-platform/test/controls/enroot_spec.rb

@@ -34,6 +34,6 @@
 
   describe 'enroot service should be disabled' do
     subject { command("enroot version") }
-    its('exit_status') { should eq 127 }
+    its('exit_status') { should cmp > 0 }
   end
 end

cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb

@@ -18,7 +18,7 @@
 return unless nvidia_enabled?
 
 pyxis_version = node['cluster']['pyxis']['version']
-pyxis_url = "https://github.com/NVIDIA/pyxis/archive/refs/tags/v#{pyxis_version}.tar.gz"
+pyxis_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/pyxis/v#{pyxis_version}.tar.gz"
 pyxis_tarball = "#{node['cluster']['sources_dir']}/pyxis-#{pyxis_version}.tar.gz"
 
 remote_file pyxis_tarball do