From 9b119c3ce13d10886002860481684c58a31d9239 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 27 Mar 2025 14:32:06 -0400 Subject: [PATCH 1/8] Add user-data.sh for Image Builder --- cli/src/pcluster/resources/imagebuilder/user_data.sh | 10 ++++++++++ cli/src/pcluster/templates/imagebuilder_stack.py | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 cli/src/pcluster/resources/imagebuilder/user_data.sh diff --git a/cli/src/pcluster/resources/imagebuilder/user_data.sh b/cli/src/pcluster/resources/imagebuilder/user_data.sh new file mode 100644 index 0000000000..0f2e65e2a6 --- /dev/null +++ b/cli/src/pcluster/resources/imagebuilder/user_data.sh @@ -0,0 +1,10 @@ +Content-Type: multipart/mixed; boundary="==BOUNDARY==" +MIME-Version: 1.0 + +--==BOUNDARY== +Content-Type: text/cloud-config; charset=us-ascii +MIME-Version: 1.0 + +package_update: false +package_upgrade: false +repo_upgrade: none \ No newline at end of file diff --git a/cli/src/pcluster/templates/imagebuilder_stack.py b/cli/src/pcluster/templates/imagebuilder_stack.py index 50b1a5b8c6..a3484bd281 100644 --- a/cli/src/pcluster/templates/imagebuilder_stack.py +++ b/cli/src/pcluster/templates/imagebuilder_stack.py @@ -395,6 +395,9 @@ def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_ tags=build_tags, parent_image=self.config.build.parent_image, components=components, + additional_instance_configuration=imagebuilder.CfnImageRecipe.AdditionalInstanceConfigurationProperty( + user_data_override=self._add_imgaebuilder_user_data_override() + ), block_device_mappings=[ imagebuilder.CfnImageRecipe.InstanceBlockDeviceMappingProperty( device_name=self._get_root_device_name(), @@ -417,6 +420,13 @@ def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_ return image_recipe_resource + def _add_imgaebuilder_user_data_override(self): + imagebuilder_user_data = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh") + with open(imagebuilder_user_data, "r", encoding="utf-8") as user_data_file: + user_data_content = user_data_file.read() + + return Fn.base64(Fn.sub(user_data_content)) + def _add_imagebuilder_components(self, build_tags, lambda_cleanup_policy_statements): imagebuilder_resources_dir = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder") From 9e6a8ad9cf6566f261523859b2894380cd02e610 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 27 Mar 2025 16:00:21 -0400 Subject: [PATCH 2/8] Add `DevSettings/DisableUserData` for Image Builder --- .../pcluster/config/imagebuilder_config.py | 2 ++ .../pcluster/schemas/imagebuilder_schema.py | 1 + .../pcluster/templates/imagebuilder_stack.py | 28 +++++++++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cli/src/pcluster/config/imagebuilder_config.py b/cli/src/pcluster/config/imagebuilder_config.py index 755aafc822..b39a636989 100644 --- a/cli/src/pcluster/config/imagebuilder_config.py +++ b/cli/src/pcluster/config/imagebuilder_config.py @@ -237,6 +237,7 @@ class ImagebuilderDevSettings(BaseDevSettings): def __init__( self, disable_pcluster_component: bool = None, + disable_user_data: bool = None, distribution_configuration: DistributionConfiguration = None, terminate_instance_on_failure: bool = None, disable_validate_and_test: bool = None, @@ -247,6 +248,7 @@ def __init__( ): super().__init__(**kwargs) self.disable_pcluster_component = Resource.init_param(disable_pcluster_component, default=False) + self.disable_user_data = Resource.init_param(disable_user_data, default=True) self.distribution_configuration = distribution_configuration self.terminate_instance_on_failure = Resource.init_param(terminate_instance_on_failure, default=True) self.disable_validate_and_test = Resource.init_param(disable_validate_and_test, default=True) diff --git a/cli/src/pcluster/schemas/imagebuilder_schema.py b/cli/src/pcluster/schemas/imagebuilder_schema.py index 1d893a12c1..ae3c858603 100644 --- a/cli/src/pcluster/schemas/imagebuilder_schema.py +++ b/cli/src/pcluster/schemas/imagebuilder_schema.py @@ -239,6 +239,7 @@ class ImagebuilderDevSettingsSchema(BaseDevSettingsSchema): """Represent the schema of the ImageBuilder Dev Setting.""" disable_pcluster_component = fields.Bool() + disable_user_data = fields.Bool() distribution_configuration = fields.Nested(DistributionConfigurationSchema) terminate_instance_on_failure = fields.Bool() disable_validate_and_test = fields.Bool() diff --git a/cli/src/pcluster/templates/imagebuilder_stack.py b/cli/src/pcluster/templates/imagebuilder_stack.py index a3484bd281..3a5e4c6b7d 100644 --- a/cli/src/pcluster/templates/imagebuilder_stack.py +++ b/cli/src/pcluster/templates/imagebuilder_stack.py @@ -385,8 +385,26 @@ def _add_imagebuilder_distribution_configuration(self, ami_tags, build_tags, lam return distribution_configuration_resource + def _add_imagebuilder_user_data_override(self): + disable_user_data = ( + self.config.dev_settings.disable_user_data + if self.config.dev_settings and self.config.dev_settings.disable_user_data + else False + ) + additional_instance_configuration_property = None + if not disable_user_data: + imagebuilder_user_data = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh") + with open(imagebuilder_user_data, "r", encoding="utf-8") as user_data_file: + user_data_content = user_data_file.read() + + additional_instance_configuration_property=imagebuilder.CfnImageRecipe.AdditionalInstanceConfigurationProperty( + user_data_override=Fn.base64(user_data_content) + ) + return additional_instance_configuration_property + def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_policy_statements): # ImageBuilderImageRecipe + image_recipe_resource = imagebuilder.CfnImageRecipe( self, "ImageRecipe", @@ -395,9 +413,7 @@ def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_ tags=build_tags, parent_image=self.config.build.parent_image, components=components, - additional_instance_configuration=imagebuilder.CfnImageRecipe.AdditionalInstanceConfigurationProperty( - user_data_override=self._add_imgaebuilder_user_data_override() - ), + additional_instance_configuration=self._add_imagebuilder_user_data_override(), block_device_mappings=[ imagebuilder.CfnImageRecipe.InstanceBlockDeviceMappingProperty( device_name=self._get_root_device_name(), @@ -420,12 +436,6 @@ def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_ return image_recipe_resource - def _add_imgaebuilder_user_data_override(self): - imagebuilder_user_data = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh") - with open(imagebuilder_user_data, "r", encoding="utf-8") as user_data_file: - user_data_content = user_data_file.read() - - return Fn.base64(Fn.sub(user_data_content)) def _add_imagebuilder_components(self, build_tags, lambda_cleanup_policy_statements): imagebuilder_resources_dir = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder") From 4d700663a6da15b16433c513be4445f1bc7bbaac Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 13:04:24 -0400 Subject: [PATCH 3/8] Add `Build/DisableKernelUpdate` for Image Builder * Add User_data which overrides package and repo update/upgrades --- .../pcluster/config/imagebuilder_config.py | 20 +++++++++++++------ .../imagebuilder/update_and_reboot.yaml | 2 +- .../pcluster/schemas/imagebuilder_schema.py | 2 -- .../pcluster/templates/imagebuilder_stack.py | 8 ++------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cli/src/pcluster/config/imagebuilder_config.py b/cli/src/pcluster/config/imagebuilder_config.py index b39a636989..00fd673418 100644 --- a/cli/src/pcluster/config/imagebuilder_config.py +++ b/cli/src/pcluster/config/imagebuilder_config.py @@ -144,6 +144,17 @@ def __init__( self.enabled = enabled +class DisableKernelUpdate(Resource): + """Represent the DisableKernelUpdate configuration for the ImageBuilder.""" + + def __init__( + self, + enabled: bool = None, + ): + super().__init__() + self.enabled = Resource.init_param(enabled, default=True) + + class LustreClient(Resource): """Represent the LustreClient configuration for the ImageBuilder.""" @@ -192,6 +203,7 @@ def __init__( security_group_ids: List[str] = None, components: List[Component] = None, update_os_packages: UpdateOsPackages = None, + disable_kernel_update: DisableKernelUpdate = None, imds: Imds = None, installation: Installation = None, ): @@ -204,6 +216,7 @@ def __init__( self.security_group_ids = security_group_ids self.components = components self.update_os_packages = update_os_packages + self.disable_kernel_update = disable_kernel_update or DisableKernelUpdate() self.imds = imds or Imds(implied="v2.0") self.installation = installation or Installation() @@ -237,23 +250,19 @@ class ImagebuilderDevSettings(BaseDevSettings): def __init__( self, disable_pcluster_component: bool = None, - disable_user_data: bool = None, distribution_configuration: DistributionConfiguration = None, terminate_instance_on_failure: bool = None, disable_validate_and_test: bool = None, cinc_installer_url: str = None, - disable_kernel_update: bool = None, slurm_patches_s3_archive: str = None, **kwargs ): super().__init__(**kwargs) self.disable_pcluster_component = Resource.init_param(disable_pcluster_component, default=False) - self.disable_user_data = Resource.init_param(disable_user_data, default=True) self.distribution_configuration = distribution_configuration self.terminate_instance_on_failure = Resource.init_param(terminate_instance_on_failure, default=True) self.disable_validate_and_test = Resource.init_param(disable_validate_and_test, default=True) self.cinc_installer_url = Resource.init_param(cinc_installer_url, default="") - self.disable_kernel_update = Resource.init_param(disable_kernel_update, default=False) self.slurm_patches_s3_archive = Resource.init_param(slurm_patches_s3_archive, default="") @@ -336,7 +345,6 @@ def __init__(self, config: ImageBuilderConfig): self.custom_node_package = None self.custom_awsbatchcli_package = None self.base_os = None - self.disable_kernel_update = None self.slurm_patches_s3_archive = None self._set_default(config) @@ -354,7 +362,7 @@ def _set_default(self, config: ImageBuilderConfig): dev_settings.slurm_patches_s3_archive if dev_settings and dev_settings.slurm_patches_s3_archive else "" ) self.base_os = "{{ build.OperatingSystemName.outputs.stdout }}" - self.disable_kernel_update = "true" if dev_settings and dev_settings.disable_kernel_update else "false" + self.disable_kernel_update = {"enabled": "yes"} if config.build.disable_kernel_update.enabled else {"enabled": "no"} for key, value in self.__dict__.items(): if not key.startswith("_") and key not in self._cluster_attributes: self._cluster_attributes.update({key: value}) diff --git a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml index d3f2549832..373e0fe1a5 100644 --- a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml +++ b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml @@ -181,7 +181,7 @@ phases: commands: - | set -v - DISABLE_KERNEL_UPDATE=$(cat /etc/parallelcluster/image_dna.json | jq -r '.cluster.disable_kernel_update') + DISABLE_KERNEL_UPDATE=$(cat /etc/parallelcluster/image_dna.json | jq -r '.cluster.disable_kernel_update.enabled') echo "${!DISABLE_KERNEL_UPDATE}" - name: PinKernelVersion diff --git a/cli/src/pcluster/schemas/imagebuilder_schema.py b/cli/src/pcluster/schemas/imagebuilder_schema.py index ae3c858603..1f6b399c12 100644 --- a/cli/src/pcluster/schemas/imagebuilder_schema.py +++ b/cli/src/pcluster/schemas/imagebuilder_schema.py @@ -239,12 +239,10 @@ class ImagebuilderDevSettingsSchema(BaseDevSettingsSchema): """Represent the schema of the ImageBuilder Dev Setting.""" disable_pcluster_component = fields.Bool() - disable_user_data = fields.Bool() distribution_configuration = fields.Nested(DistributionConfigurationSchema) terminate_instance_on_failure = fields.Bool() disable_validate_and_test = fields.Bool() cinc_installer_url = fields.Str() - disable_kernel_update = fields.Bool() @post_load def make_resource(self, data, **kwargs): diff --git a/cli/src/pcluster/templates/imagebuilder_stack.py b/cli/src/pcluster/templates/imagebuilder_stack.py index 3a5e4c6b7d..208762ac26 100644 --- a/cli/src/pcluster/templates/imagebuilder_stack.py +++ b/cli/src/pcluster/templates/imagebuilder_stack.py @@ -386,13 +386,9 @@ def _add_imagebuilder_distribution_configuration(self, ami_tags, build_tags, lam return distribution_configuration_resource def _add_imagebuilder_user_data_override(self): - disable_user_data = ( - self.config.dev_settings.disable_user_data - if self.config.dev_settings and self.config.dev_settings.disable_user_data - else False - ) + additional_instance_configuration_property = None - if not disable_user_data: + if self.config.build and self.config.build.disable_kernel_update and self.config.build.disable_kernel_update.enabled: imagebuilder_user_data = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh") with open(imagebuilder_user_data, "r", encoding="utf-8") as user_data_file: user_data_content = user_data_file.read() From 2e76c89df8cf8506335a371a4356d196e27b5c73 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 17:49:48 -0400 Subject: [PATCH 4/8] Update unit test --- .../pcluster/config/dummy_imagebuilder_config.py | 3 ++- cli/tests/pcluster/models/test_imagebuilder.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cli/tests/pcluster/config/dummy_imagebuilder_config.py b/cli/tests/pcluster/config/dummy_imagebuilder_config.py index 90f6e52222..bc5be2ab64 100644 --- a/cli/tests/pcluster/config/dummy_imagebuilder_config.py +++ b/cli/tests/pcluster/config/dummy_imagebuilder_config.py @@ -23,7 +23,7 @@ LustreClient, NvidiaSoftware, UpdateOsPackages, - Volume, + Volume, DisableKernelUpdate, ) CLASS_DICT = { @@ -41,6 +41,7 @@ "iam": Iam, "additional_iam_policies": AdditionalIamPolicy, "update_os_packages": UpdateOsPackages, + "disable_kernel_update": DisableKernelUpdate, "imds": Imds, "installation": Installation, "lustre_client": LustreClient, diff --git a/cli/tests/pcluster/models/test_imagebuilder.py b/cli/tests/pcluster/models/test_imagebuilder.py index 4e467e7e60..9348a35836 100644 --- a/cli/tests/pcluster/models/test_imagebuilder.py +++ b/cli/tests/pcluster/models/test_imagebuilder.py @@ -162,11 +162,11 @@ def test_imagebuilder_url_validator( "parent_image": "ami-0185634c5a8a37250", "instance_type": "c5.xlarge", "update_os_packages": {"enabled": True}, + "disable_kernel_update": {"enabled": True}, }, "dev_settings": { "node_package": "s3://test/aws-parallelcluster-node-3.0.tgz", "aws_batch_cli_package": "https://test/aws-parallelcluster-3.0.tgz", - "disable_kernel_update": "true", }, } }, @@ -175,7 +175,7 @@ def test_imagebuilder_url_validator( "base_os": "{{ build.OperatingSystemName.outputs.stdout }}", "custom_awsbatchcli_package": "https://test/aws-parallelcluster-3.0.tgz", "custom_node_package": "s3://test/aws-parallelcluster-node-3.0.tgz", - "disable_kernel_update": "true", + "disable_kernel_update": {"enabled": "yes"}, "is_official_ami_build": "false", "nvidia": {"enabled": "no"}, "lustre": {"enabled": "yes"}, @@ -207,7 +207,7 @@ def test_imagebuilder_url_validator( "custom_awsbatchcli_package": "", "custom_node_package": "s3://test/aws-parallelcluster-node-3.0.tgz", "dcv": "no", - "disable_kernel_update": "false", + "disable_kernel_update": {"enabled": "yes"}, "is_official_ami_build": "false", "lustre": {"enabled": "no"}, "nvidia": {"enabled": "yes"}, @@ -223,6 +223,7 @@ def test_imagebuilder_url_validator( "parent_image": "ami-0185634c5a8a37250", "instance_type": "c5.xlarge", "installation": {"lustre_client": {"enabled": True}}, + "disable_kernel_update": {"enabled": False}, }, "dev_settings": { "cookbook": { @@ -240,7 +241,7 @@ def test_imagebuilder_url_validator( "custom_awsbatchcli_package": "https://test/aws-parallelcluster-3.0.tgz", "custom_node_package": "", "dcv": "no", - "disable_kernel_update": "false", + "disable_kernel_update": {"enabled": "no"}, "is_official_ami_build": "false", "nvidia": {"enabled": "yes"}, "lustre": {"enabled": "yes"}, @@ -272,7 +273,7 @@ def test_imagebuilder_url_validator( "base_os": "{{ build.OperatingSystemName.outputs.stdout }}", "custom_awsbatchcli_package": "https://test/aws-parallelcluster-3.0.tgz", "custom_node_package": "", - "disable_kernel_update": "false", + "disable_kernel_update": {"enabled": "yes"}, "is_official_ami_build": "true", "nvidia": {"enabled": "yes"}, "lustre": {"enabled": "yes"}, @@ -289,6 +290,7 @@ def test_imagebuilder_url_validator( "build": { "parent_image": "ami-0185634c5a8a37250", "instance_type": "c5.xlarge", + "disable_kernel_update": {"enabled": True}, "installation": {"nvidia_software": {"enabled": False}}, }, "dev_settings": { @@ -307,7 +309,7 @@ def test_imagebuilder_url_validator( "base_os": "{{ build.OperatingSystemName.outputs.stdout }}", "custom_awsbatchcli_package": "", "custom_node_package": "", - "disable_kernel_update": "false", + "disable_kernel_update": {"enabled": "yes"}, "is_official_ami_build": "false", "nvidia": {"enabled": "no"}, "lustre": {"enabled": "yes"}, From f5ff513226630072199d68c7b50485ecc642503c Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 18:03:00 -0400 Subject: [PATCH 5/8] Adding Kernel Pinning in parallelcluster.yaml --- .../imagebuilder/parallelcluster.yaml | 84 +++++++++++++++++-- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index 1465309e57..ef959016ec 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -182,6 +182,54 @@ phases: fi fi + - name: CreatingJsonFile + action: CreateFile + inputs: + - path: /etc/parallelcluster/image_dna.json + content: | + ${CfnParamChefDnaJson} + overwrite: true + + - name: DisableKernelUpdate + action: ExecuteBash + inputs: + commands: + - | + set -v + DISABLE_KERNEL_UPDATE=$(cat /etc/parallelcluster/image_dna.json | jq -r '.cluster.disable_kernel_update.enabled') + echo "${!DISABLE_KERNEL_UPDATE}" + + - name: PinKernelVersion + action: ExecuteBash + inputs: + commands: + - | + set -v + OS='{{ build.OperatingSystemName.outputs.stdout }}' + PLATFORM='{{ build.PlatformName.outputs.stdout }}' + DISABLE_KERNEL_UPDATE='{{ build.DisableKernelUpdate.outputs.stdout }}' + + if [[ ${!DISABLE_KERNEL_UPDATE} == true ]]; then + if [[ ${!PLATFORM} == RHEL ]]; then + yum install -y yum-plugin-versionlock + # listing all the packages because wildcard does not work as expected + yum versionlock kernel kernel-core kernel-modules + + if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then + yum versionlock kernel-abi-whitelists + else + yum versionlock kernel-abi-stablelists + fi + + if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then + yum versionlock rocky-release rocky-repos + elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then + yum versionlock redhat-release + fi + echo "Kernel version locked" + fi + fi + # Install prerequisite OS packages - name: InstallPrerequisite action: ExecuteBash @@ -282,14 +330,6 @@ phases: cookbook_path ['/etc/chef/cookbooks'] overwrite: true - - name: CreatingJsonFile - action: CreateFile - inputs: - - path: /etc/parallelcluster/image_dna.json - content: | - ${CfnParamChefDnaJson} - overwrite: true - - name: InstallPClusterPackages action: ExecuteBash inputs: @@ -308,6 +348,34 @@ phases: {{ build.PClusterCookbookVersionName.outputs.stdout }} overwrite: true + - name: RemoveKernelPin + action: ExecuteBash + inputs: + commands: + - | + set -v + OS='{{ build.OperatingSystemName.outputs.stdout }}' + PLATFORM='{{ build.PlatformName.outputs.stdout }}' + DISABLE_KERNEL_UPDATE='{{ build.DisableKernelUpdate.outputs.stdout }}' + + # Remove kernel version lock + if [[ ${!DISABLE_KERNEL_UPDATE} == true ]] && [[ ${!PLATFORM} == RHEL ]]; then + yum versionlock delete kernel kernel-core kernel-modules + + if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then + yum versionlock delete kernel-abi-whitelists + else + yum versionlock delete kernel-abi-stablelists + fi + + if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then + yum versionlock delete rocky-release + elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then + yum versionlock delete redhat-release + fi + echo "Kernel version unlocked" + fi + - name: KeepSSM action: ExecuteBash inputs: From 7dbd12d134eaa9c9060d80bd6e4ed48f39eaa9d7 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 18:09:34 -0400 Subject: [PATCH 6/8] Removing centos usage in ImageBuilder --- .../resources/imagebuilder/parallelcluster.yaml | 10 ++-------- .../imagebuilder/parallelcluster_validate.yaml | 2 +- .../resources/imagebuilder/update_and_reboot.yaml | 4 ++-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index ef959016ec..42b781a331 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -129,7 +129,7 @@ phases: set -v OS='{{ build.OperatingSystemName.outputs.stdout }}' - if [ `echo "${!OS}" | grep -E '^(alinux|centos|rhel|rocky)'` ]; then + if [ `echo "${!OS}" | grep -E '^(alinux|rhel|rocky)'` ]; then PLATFORM='RHEL' elif [ `echo "${!OS}" | grep -E '^ubuntu'` ]; then PLATFORM='DEBIAN' @@ -167,7 +167,7 @@ phases: set -v if [ ${CfnParamUpdateOsAndReboot} == false ]; then RELEASE='{{ build.OperatingSystemRelease.outputs.stdout }}' - if [ `echo "${!RELEASE}" | grep -Ev '^(amzn|centos|ubuntu|rhel|rocky)'` ]; then + if [ `echo "${!RELEASE}" | grep -Ev '^(amzn|ubuntu|rhel|rocky)'` ]; then echo "This component does not support '${!RELEASE}'. Failing build." exit {{ FailExitCode }} fi @@ -251,12 +251,6 @@ phases: fi yum -y update krb5-libs yum -y groupinstall development && sudo yum -y install curl wget jq - - - if [[ ${!OS} =~ ^centos ]]; then - /bin/sed -r -i -e 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config - grub2-mkconfig -o /boot/grub2/grub.cfg - fi elif [[ ${!PLATFORM} == DEBIAN ]]; then if [[ "${CfnParamUpdateOsAndReboot}" == "false" ]]; then # disable apt-daily.timer to avoid dpkg lock diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster_validate.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster_validate.yaml index 08f0606812..6589a97d5a 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster_validate.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster_validate.yaml @@ -85,7 +85,7 @@ phases: set -v OS='{{ validate.OperatingSystemName.outputs.stdout }}' - if [ `echo "${OS}" | grep -E '^(alinux|centos|rhel|rocky)'` ]; then + if [ `echo "${OS}" | grep -E '^(alinux|rhel|rocky)'` ]; then PLATFORM='RHEL' elif [ `echo "${OS}" | grep -E '^ubuntu'` ]; then PLATFORM='DEBIAN' diff --git a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml index 373e0fe1a5..036e9daa90 100644 --- a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml +++ b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml @@ -67,7 +67,7 @@ phases: set -v OS='{{ build.OperatingSystemName.outputs.stdout }}' - if [ `echo "${!OS}" | grep -E '^(alinux|centos|rhel|rocky)'` ]; then + if [ `echo "${!OS}" | grep -E '^(alinux|rhel|rocky)'` ]; then PLATFORM='RHEL' elif [ `echo "${!OS}" | grep -E '^ubuntu'` ]; then PLATFORM='DEBIAN' @@ -83,7 +83,7 @@ phases: - | set -v RELEASE='{{ build.OperatingSystemRelease.outputs.stdout }}' - if [ `echo "${!RELEASE}" | grep -Ev '^(amzn|centos|ubuntu|rhel|rocky)'` ]; then + if [ `echo "${!RELEASE}" | grep -Ev '^(amzn|ubuntu|rhel|rocky)'` ]; then echo "This component does not support '${!RELEASE}'. Failing build." exit {{ FailExitCode }} fi From 99fedbcedadf176fce060998a9dab85a5c9b24df Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 18:11:43 -0400 Subject: [PATCH 7/8] code formatting --- cli/src/pcluster/config/imagebuilder_config.py | 8 +++++--- .../pcluster/templates/imagebuilder_stack.py | 17 ++++++++++++----- .../config/dummy_imagebuilder_config.py | 3 ++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cli/src/pcluster/config/imagebuilder_config.py b/cli/src/pcluster/config/imagebuilder_config.py index 00fd673418..66d6ff8ea3 100644 --- a/cli/src/pcluster/config/imagebuilder_config.py +++ b/cli/src/pcluster/config/imagebuilder_config.py @@ -148,8 +148,8 @@ class DisableKernelUpdate(Resource): """Represent the DisableKernelUpdate configuration for the ImageBuilder.""" def __init__( - self, - enabled: bool = None, + self, + enabled: bool = None, ): super().__init__() self.enabled = Resource.init_param(enabled, default=True) @@ -362,7 +362,9 @@ def _set_default(self, config: ImageBuilderConfig): dev_settings.slurm_patches_s3_archive if dev_settings and dev_settings.slurm_patches_s3_archive else "" ) self.base_os = "{{ build.OperatingSystemName.outputs.stdout }}" - self.disable_kernel_update = {"enabled": "yes"} if config.build.disable_kernel_update.enabled else {"enabled": "no"} + self.disable_kernel_update = ( + {"enabled": "yes"} if config.build.disable_kernel_update.enabled else {"enabled": "no"} + ) for key, value in self.__dict__.items(): if not key.startswith("_") and key not in self._cluster_attributes: self._cluster_attributes.update({key: value}) diff --git a/cli/src/pcluster/templates/imagebuilder_stack.py b/cli/src/pcluster/templates/imagebuilder_stack.py index 208762ac26..e29d43104c 100644 --- a/cli/src/pcluster/templates/imagebuilder_stack.py +++ b/cli/src/pcluster/templates/imagebuilder_stack.py @@ -388,13 +388,21 @@ def _add_imagebuilder_distribution_configuration(self, ami_tags, build_tags, lam def _add_imagebuilder_user_data_override(self): additional_instance_configuration_property = None - if self.config.build and self.config.build.disable_kernel_update and self.config.build.disable_kernel_update.enabled: - imagebuilder_user_data = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh") + if ( + self.config.build + and self.config.build.disable_kernel_update + and self.config.build.disable_kernel_update.enabled + ): + imagebuilder_user_data = os.path.join( + imagebuilder_utils.get_resources_directory(), "imagebuilder", "user_data.sh" + ) with open(imagebuilder_user_data, "r", encoding="utf-8") as user_data_file: user_data_content = user_data_file.read() - additional_instance_configuration_property=imagebuilder.CfnImageRecipe.AdditionalInstanceConfigurationProperty( - user_data_override=Fn.base64(user_data_content) + additional_instance_configuration_property = ( + imagebuilder.CfnImageRecipe.AdditionalInstanceConfigurationProperty( + user_data_override=Fn.base64(user_data_content) + ) ) return additional_instance_configuration_property @@ -432,7 +440,6 @@ def _add_imagebuilder_image_recipe(self, build_tags, components, lambda_cleanup_ return image_recipe_resource - def _add_imagebuilder_components(self, build_tags, lambda_cleanup_policy_statements): imagebuilder_resources_dir = os.path.join(imagebuilder_utils.get_resources_directory(), "imagebuilder") diff --git a/cli/tests/pcluster/config/dummy_imagebuilder_config.py b/cli/tests/pcluster/config/dummy_imagebuilder_config.py index bc5be2ab64..b42e1b5f2e 100644 --- a/cli/tests/pcluster/config/dummy_imagebuilder_config.py +++ b/cli/tests/pcluster/config/dummy_imagebuilder_config.py @@ -13,6 +13,7 @@ AdditionalIamPolicy, Build, Component, + DisableKernelUpdate, DistributionConfiguration, Iam, Image, @@ -23,7 +24,7 @@ LustreClient, NvidiaSoftware, UpdateOsPackages, - Volume, DisableKernelUpdate, + Volume, ) CLASS_DICT = { From 75273e4af54386f9594822df5b2787f044109b14 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 1 May 2025 20:15:42 -0400 Subject: [PATCH 8/8] add disable kernel Update in schema --- cli/src/pcluster/schemas/imagebuilder_schema.py | 13 +++++++++++++ .../imagebuilder_schema_all.yaml | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cli/src/pcluster/schemas/imagebuilder_schema.py b/cli/src/pcluster/schemas/imagebuilder_schema.py index 1f6b399c12..c77c8075c1 100644 --- a/cli/src/pcluster/schemas/imagebuilder_schema.py +++ b/cli/src/pcluster/schemas/imagebuilder_schema.py @@ -20,6 +20,7 @@ from pcluster.config.imagebuilder_config import ( Build, Component, + DisableKernelUpdate, DistributionConfiguration, Iam, Image, @@ -170,6 +171,17 @@ def make_resource(self, data, **kwargs): return UpdateOsPackages(**data) +class DisableKernelUpdateSchema(BaseSchema): + """Represent the schema of the ImageBuilder DisableKernelUpdate.""" + + enabled = fields.Bool() + + @post_load + def make_resource(self, data, **kwargs): + """Generate resource.""" + return DisableKernelUpdate(**data) + + class LustreClientSchema(BaseSchema): """Represent the schema of the ImageBuilder NvidiaSoftware.""" @@ -215,6 +227,7 @@ class BuildSchema(BaseSchema): security_group_ids = fields.List(fields.Str) subnet_id = fields.Str(validate=get_field_validator("subnet_id")) update_os_packages = fields.Nested(UpdateOsPackagesSchema) + disable_kernel_update = fields.Nested(DisableKernelUpdateSchema) imds = fields.Nested(ImdsSchema) installation = fields.Nested(InstallationSchema) diff --git a/cli/tests/pcluster/schemas/test_imagebuilder_schema/test_imagebuilder_schema/imagebuilder_schema_all.yaml b/cli/tests/pcluster/schemas/test_imagebuilder_schema/test_imagebuilder_schema/imagebuilder_schema_all.yaml index 04280a53cc..86ff67ae4d 100644 --- a/cli/tests/pcluster/schemas/test_imagebuilder_schema/test_imagebuilder_schema/imagebuilder_schema_all.yaml +++ b/cli/tests/pcluster/schemas/test_imagebuilder_schema/test_imagebuilder_schema/imagebuilder_schema_all.yaml @@ -37,6 +37,8 @@ Build: SubnetId: subnet-0d03dc52 UpdateOsPackages: Enabled: true + DisableKernelUpdate: + Enabled: true Installation: NvidiaSoftware: Enabled: true @@ -57,7 +59,6 @@ DevSettings: { "UserIds": ["123456789012", "345678901234"], "UserGroups": ["all"] } TerminateInstanceOnFailure: True DisableValidateAndTest: True - DisableKernelUpdate: True DeploymentSettings: {% if disable_sudo_access_for_default_user == "True" %}DisableSudoAccessForDefaultUser: True{% endif %}