[Develop] Introduce Global Cleanup IAM Role for ParallelCluster Build-Image (#6912)

* feat: Global Cleanup IAM Role for ParallelCluster Build-Image

* Introduce ensure_cleanup_role() with bootrapped tagging and idempotent creation / update logic.
  * 4-step safe update sequence documented, now only after the inline policy succeeds, set or bump the bootrapped tag.
  * Lambda VPCAccess managed policy is attached only when LambdaFunctionsVpcConfig exists in the config
* Modify image_operations_controller to invoke ensure_cleanup_role when Build/Iam/CleanupLambdaRole is not provided and to fail fast on permission errors.
* Refactor imagebuilder_stack to remove all per-stack cleanup-role logic and wire Lambda to the global role by default.
* Update constants (role prefix / expected revision tag key & value).
* IamClient – add create_role, attach_role_policy, put_role_policy, tag_role

Author: hehe7318

CHANGELOG.md

@@ -10,6 +10,8 @@ CHANGELOG
 - Support DCV on Amazon Linux 2023.
 - Upgrade Python runtime used by Lambda functions to python3.12 (from python3.9).
 - Remove `berkshelf`. All cookbooks are local and do not need `berkshelf` dependency management.
+- The build-image command now deploys a global role that is used to automatically delete the build-image stack after images either succeed or fail the build. 
+  The role is meant to exists even after the stack has been deleted. This is to prevent build-image stack deletion failures, reported in https://github.com/aws/aws-parallelcluster/issues/5914
 - Add the configuration parameter `HeadNode/SharedStorageEfsSettings/Encrypted` to enable encryption on the EFS file system used for the head node internal shared storage.
 
 **BUG FIXES**

cli/src/pcluster/api/controllers/image_operations_controller.py

@@ -10,6 +10,8 @@
 import logging
 import os as os_lib
 
+import yaml
+
 from pcluster.api.controllers.common import (
     assert_supported_operation,
     configure_aws_region,
@@ -52,14 +54,15 @@
 from pcluster.aws.common import AWSClientError
 from pcluster.aws.ec2 import Ec2Client
 from pcluster.constants import SUPPORTED_ARCHITECTURES, SUPPORTED_OSES, Operation
+from pcluster.imagebuilder_utils import ensure_default_build_image_stack_cleanup_role
 from pcluster.models.imagebuilder import (
     BadRequestImageBuilderActionError,
     ConfigValidationError,
     ImageBuilder,
     NonExistingImageError,
 )
 from pcluster.models.imagebuilder_resources import ImageBuilderStack, NonExistingStackError
-from pcluster.utils import get_installed_version, to_utc_datetime
+from pcluster.utils import get_installed_version, get_partition, to_utc_datetime
 from pcluster.validators.common import FailureLevel
 
 LOGGER = logging.getLogger(__name__)
@@ -105,6 +108,29 @@ def build_image(
     validation_failure_level = validation_failure_level or ValidationLevel.ERROR
     dryrun = dryrun or False
 
+    raw_cfg_str = build_image_request_content["imageConfiguration"]
+    cfg_dict = yaml.safe_load(raw_cfg_str) or {}
+    # If CleanupLambdaRole exists in the config, skip ensure_default_build_image_stack_cleanup_role
+    has_custom_cleanup_role = cfg_dict.get("Build", {}).get("Iam", {}).get("CleanupLambdaRole")
+
+    if not has_custom_cleanup_role:
+        try:
+            # If LambdaFunctionsVpcConfig exists in the config, attach the AWS-managed LambdaVPCAccess policy
+            has_lambda_functions_vpc_config = cfg_dict.get("DeploymentSettings", {}).get("LambdaFunctionsVpcConfig")
+            account_id = AWSApi.instance().sts.get_account_id()
+            ensure_default_build_image_stack_cleanup_role(
+                account_id, get_partition(), attach_vpc_access_policy=bool(has_lambda_functions_vpc_config)
+            )
+        except AWSClientError as e:
+            if e.error_code in ("AccessDenied", "AccessDeniedException", "UnauthorizedOperation"):
+                raise BadRequestException(
+                    "Current principal lacks permissions to create or update the ParallelCluster build-image "
+                    "cleanup IAM role. "
+                    "Either pass `Build/Iam/CleanupLambdaRole` or grant the missing permissions to continue. "
+                    "For detailed instructions, please refer to our public documentation."
+                )
+            raise
+
     build_image_request_content = BuildImageRequestContent.from_dict(build_image_request_content)
 
     try:

cli/src/pcluster/aws/iam.py

@@ -32,3 +32,23 @@ def get_role(self, role_name):
     def get_instance_profile(self, instance_profile_name):
         """Get instance profile information."""
         return self._client.get_instance_profile(InstanceProfileName=instance_profile_name)
+
+    @AWSExceptionHandler.handle_client_exception
+    def create_role(self, **kwargs):
+        """Create IAM role."""
+        return self._client.create_role(**kwargs)
+
+    @AWSExceptionHandler.handle_client_exception
+    def attach_role_policy(self, role_name, policy_arn):
+        """Attach a managed policy to the given role."""
+        return self._client.attach_role_policy(RoleName=role_name, PolicyArn=policy_arn)
+
+    @AWSExceptionHandler.handle_client_exception
+    def put_role_policy(self, role_name, policy_name, policy_document):
+        """Create or replace the specified inline policy on a role."""
+        return self._client.put_role_policy(RoleName=role_name, PolicyName=policy_name, PolicyDocument=policy_document)
+
+    @AWSExceptionHandler.handle_client_exception
+    def tag_role(self, role_name, tags):
+        """Add or overwrite one or more tags for the specified role."""
+        return self._client.tag_role(RoleName=role_name, Tags=tags)

cli/src/pcluster/constants.py

@@ -335,3 +335,8 @@ class Operation(Enum):
 PCLUSTER_BUCKET_PROTECTED_FOLDER = "parallelcluster"
 PCLUSTER_BUCKET_PROTECTED_PREFIX = f"{PCLUSTER_BUCKET_PROTECTED_FOLDER}/"
 PCLUSTER_BUCKET_REQUIRED_BOOTSTRAP_FEATURES = ["basic", "export-logs"]
+
+PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_PREFIX = "PClusterBuildImageCleanupRole"
+# Tag key & expected revision (increment when policy widens)
+PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_REVISION = 1
+PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_BOOTSTRAP_TAG_KEY = "parallelcluster:build-image-cleanup-role-bootstrapped"

cli/src/pcluster/imagebuilder_utils.py

@@ -8,12 +8,21 @@
 # or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
+import json
+import logging
 import os
 
 import yaml
 
 from pcluster.aws.aws_api import AWSApi
-from pcluster.utils import get_url_scheme, yaml_load
+from pcluster.aws.common import AWSClientError
+from pcluster.constants import (
+    IAM_ROLE_PATH,
+    PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_BOOTSTRAP_TAG_KEY,
+    PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_PREFIX,
+    PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_REVISION,
+)
+from pcluster.utils import generate_string_hash, get_url_scheme, yaml_load
 
 ROOT_VOLUME_TYPE = "gp3"
 PCLUSTER_RESERVED_VOLUME_SIZE = 37
@@ -65,3 +74,174 @@ def _generate_action(action_name, commands):
     """Generate action in imagebuilder components."""
     action = {"name": action_name, "action": "ExecuteBash", "inputs": {"commands": [commands]}}
     return action
+
+
+def get_cleanup_role_name(account_id: str) -> str:
+    """Return the role name including a revision number."""
+    hashed_account_id = generate_string_hash(account_id)
+    return (
+        f"{PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_PREFIX}-{hashed_account_id}-v{PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_REVISION}"
+    )
+
+
+def _expected_inline_policy(account_id: str, partition: str):
+    """Return the inline policy document (JSON-serialised string)."""
+    return json.dumps(
+        {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Action": ["iam:DetachRolePolicy", "iam:DeleteRole", "iam:DeleteRolePolicy"],
+                    "Resource": f"arn:{partition}:iam::{account_id}:role/parallelcluster/*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": ["iam:DeleteInstanceProfile", "iam:RemoveRoleFromInstanceProfile"],
+                    "Resource": f"arn:{partition}:iam::{account_id}:instance-profile/parallelcluster/*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": "imagebuilder:DeleteInfrastructureConfiguration",
+                    "Resource": f"arn:{partition}:imagebuilder:*:{account_id}:infrastructure-configuration/"
+                    f"parallelclusterimage-*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": ["imagebuilder:DeleteComponent"],
+                    "Resource": [f"arn:{partition}:imagebuilder:*:{account_id}:component/parallelclusterimage-*/*"],
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": "imagebuilder:DeleteImageRecipe",
+                    "Resource": f"arn:{partition}:imagebuilder:*:{account_id}:image-recipe/parallelclusterimage-*/*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": "imagebuilder:DeleteDistributionConfiguration",
+                    "Resource": f"arn:{partition}:imagebuilder:*:{account_id}:distribution-configuration/"
+                    f"parallelclusterimage-*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": ["imagebuilder:DeleteImage", "imagebuilder:GetImage", "imagebuilder:CancelImageCreation"],
+                    "Resource": f"arn:{partition}:imagebuilder:*:{account_id}:image/parallelclusterimage-*/*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": "cloudformation:DeleteStack",
+                    "Resource": f"arn:{partition}:cloudformation:*:{account_id}:stack/*/*",
+                    "Condition": {
+                        "ForAnyValue:StringLike": {"cloudformation:ResourceTag/parallelcluster:image_id": "*"}
+                    },
+                    "Effect": "Allow",
+                },
+                # The below two permissions are required for the DeleteStackFunction Lambda to tag the
+                # created AMI with 'parallelcluster:build_status' and 'parallelcluster:parent_image' tags
+                {"Action": "ec2:CreateTags", "Resource": f"arn:{partition}:ec2:*::image/*", "Effect": "Allow"},
+                {"Action": "tag:TagResources", "Resource": "*", "Effect": "Allow"},
+                {
+                    "Action": ["lambda:DeleteFunction", "lambda:RemovePermission"],
+                    "Resource": f"arn:{partition}:lambda:*:{account_id}:function:ParallelClusterImage-*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": "logs:DeleteLogGroup",
+                    "Resource": f"arn:{partition}:logs:*:{account_id}:log-group:/aws/lambda/ParallelClusterImage-*:*",
+                    "Effect": "Allow",
+                },
+                {
+                    "Action": [
+                        "SNS:GetTopicAttributes",
+                        "SNS:DeleteTopic",
+                        "SNS:GetSubscriptionAttributes",
+                        "SNS:Unsubscribe",
+                    ],
+                    "Resource": f"arn:{partition}:sns:*:{account_id}:ParallelClusterImage-*",
+                    "Effect": "Allow",
+                },
+            ],
+        }
+    )
+
+
+def ensure_default_build_image_stack_cleanup_role(
+    account_id: str, partition="aws", attach_vpc_access_policy: bool = False
+) -> str:
+    """
+    Ensure the global (account-wide) cleanup role exists and is at the expected revision.
+
+    The function follows a safe order:
+      1. If the role does not exist, create it without the bootstrapped tag.
+      2. If LambdaFunctionsVpcConfig exists in the config, attach the AWS-managed LambdaVPCAccess policy.
+      3. Attach the AWS-managed Lambda basic policy.
+      4. Update/write the inline policy (least-privilege cleanup policy).
+      5. Only after the inline policy succeeds, set the bootstrapped tag.
+
+    This way, if step 2, 3 or 4 fails (e.g., lack of iam:PutRolePolicy permission),
+    future invocations will keep retrying.
+    """
+    iam = AWSApi.instance().iam
+    role_name = get_cleanup_role_name(account_id)
+    role_arn = f"arn:{partition}:iam::{account_id}:role{IAM_ROLE_PATH}{role_name}"
+
+    # Assume-role trust policy
+    assume_doc = {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": {"Service": "lambda.amazonaws.com"},
+                "Action": "sts:AssumeRole",
+                "Condition": {
+                    "ArnLike": {
+                        "aws:SourceArn": f"arn:{partition}:lambda:*:{account_id}:function:ParallelClusterImage-*"
+                    }
+                },
+            }
+        ],
+    }
+    # Check whether the role already exists
+    try:
+        resp = iam.get_role(role_name=role_name)
+        tags = {t["Key"]: t["Value"] for t in resp["Role"].get("Tags", [])}
+        already_bootstrapped = tags.get(PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_BOOTSTRAP_TAG_KEY, "").lower() == "true"
+    except AWSClientError as e:
+        if e.error_code == "NoSuchEntity":
+            logging.info("Creating default build-image stack cleanup role %s because it does not exists.", role_name)
+            iam.create_role(
+                RoleName=role_name,
+                Path=IAM_ROLE_PATH,
+                AssumeRolePolicyDocument=json.dumps(assume_doc),
+                Description="AWS ParallelCluster build-image cleanup Lambda execution role. Please do not delete it.",
+            )
+            already_bootstrapped = False
+        else:
+            raise
+
+    # Attach AWSLambdaVPCAccessExecutionRole
+    if attach_vpc_access_policy:
+        iam.attach_role_policy(
+            role_name,
+            f"arn:{partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole",
+        )
+
+    if already_bootstrapped:
+        return role_arn
+
+    # Attach AWSLambdaBasicExecutionRole
+    cleanup_role_basic_managed_policy = f"arn:{partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+    iam.attach_role_policy(role_name, cleanup_role_basic_managed_policy)
+
+    # Put inline policy
+    iam.put_role_policy(
+        role_name=role_name,
+        policy_name="ParallelClusterCleanupInline",
+        policy_document=_expected_inline_policy(account_id, partition),
+    )
+
+    # Set bootstrapped tag after policy write succeeds
+    iam.tag_role(
+        role_name=role_name,
+        tags=[{"Key": PCLUSTER_BUILD_IMAGE_CLEANUP_ROLE_BOOTSTRAP_TAG_KEY, "Value": "true"}],
+    )
+    return role_arn