Module resource foreach

Author: ericzbeard

awscli/customizations/cloudformation/modules/flatten.py

@@ -22,7 +22,6 @@
 from awscli.customizations.cloudformation import exceptions
 from awscli.customizations.cloudformation.modules.visitor import Visitor
 from awscli.customizations.cloudformation.modules.merge import isdict
-from awscli.customizations.cloudformation.yamlhelper import yaml_dump
 
 LOG = logging.getLogger(__name__)
 
@@ -567,8 +566,6 @@ def _process_flatten(flatten_config):
         InvalidModuleError: If the configuration is invalid
     """
 
-    print("_process_flatten:", yaml_dump(flatten_config))
-
     # Handle simple source case (just a list or scalar)
     if not isdict(flatten_config):
         if isinstance(flatten_config, list):
@@ -647,8 +644,6 @@ def vf(v):
             flatten_config = v.d[FLATTEN]
             result = _process_flatten(flatten_config)
 
-            print("flatten result:", yaml_dump(result))
-
             # Replace the Fn::Flatten with its result
             v.p[v.k] = result
         except Exception as e:

awscli/customizations/cloudformation/modules/foreach.py

@@ -33,7 +33,6 @@
 from awscli.customizations.cloudformation.modules.util import (
     isdict,
 )
-from awscli.customizations.cloudformation.yamlhelper import yaml_dump
 
 IDENTIFIER_PLACEHOLDER = "$Identifier"
 INDEX_PLACEHOLDER = "$Index"
@@ -187,9 +186,6 @@ def process_foreach_item(
     4. A Map where keys serve as identifiers
     """
 
-    print(f"Processing ForEach for {name}")
-    print(config)
-
     # Modules or Resources
     items = template[section]
 
@@ -213,8 +209,6 @@ def _resolve_foreach_ref(config, k, parent_module):
 
     fe = config[FOREACH]
 
-    print("_resolve_foreach_ref fe:", yaml_dump(fe))
-
     if isdict(fe):
         if REF in fe:
             resolved = parent_module.find_ref(fe[REF])
@@ -226,16 +220,10 @@ def _resolve_foreach_ref(config, k, parent_module):
 
         if FLATTEN in fe:
 
-            print("fe before resolve:", yaml_dump(fe))
-
             parent_module.resolve(fe)
 
-            print("fe after resolve:", yaml_dump(fe))
-
             fn_flatten(config)
 
-            print("config after flatten:", yaml_dump(config))
-
             return
 
         # Validate map keys
@@ -252,7 +240,6 @@ def _parse_foreach_value(m, k):
 
     # Handle empty list case
     if isinstance(m, list) and not m:
-        print(f"Warning: ForEach for {k} is an empty list")
         return tokens, values
 
     # List of objects with Identifier property
@@ -336,6 +323,30 @@ def _create_foreach_copies(name, config, tokens, values, items):
     del items[name]
 
 
+def _get_value_from_getatt_reference(node_dict, value_obj):
+    """
+    Extract a value from a GetAtt reference to $Value.X
+
+    node_dict: The dictionary that might contain a GetAtt reference
+    value_obj: The value object containing properties to reference
+
+    Returns: The referenced value if found, None otherwise
+    """
+    if not isinstance(node_dict, dict) or GETATT not in node_dict:
+        return None
+
+    getatt = node_dict[GETATT]
+    if not (isinstance(getatt, list) and len(getatt) >= 2):
+        return None
+
+    if getatt[0] == "$Value" and isinstance(value_obj, dict):
+        prop_name = getatt[1]
+        if prop_name in value_obj:
+            return value_obj[prop_name]
+
+    return None
+
+
 def _process_properties(module, i, token):
     """
     Process properties in a copied module, replacing placeholders.
@@ -365,14 +376,12 @@ def resolve_identifier_refs(vis):
                     else:
                         vis.p[vis.k] = new_val
             # Handle GetAtt to $Value.X
-            elif isinstance(vis.d, dict) and GETATT in vis.d:
-                getatt = vis.d[GETATT]
-                if isinstance(getatt, list) and len(getatt) >= 2:
-                    if getatt[0] == "$Value" and FOREACH_VALUE in module:
-                        value_obj = module[FOREACH_VALUE]
-                        prop_name = getatt[1]
-                        if prop_name in value_obj:
-                            vis.p[vis.k] = value_obj[prop_name]
+            elif FOREACH_VALUE in module:
+                value = _get_value_from_getatt_reference(
+                    vis.d, module[FOREACH_VALUE]
+                )
+                if value is not None:
+                    vis.p[vis.k] = value
 
     # Process ${Identifier} references
     Visitor(module[PROPERTIES]).visit(resolve_identifier_refs)
@@ -619,14 +628,15 @@ def resolve_value_refs(v):
                 modified = False
 
                 # Replace ${Value.X} with actual values
-                for prop_name, prop_value in value_obj.items():
-                    if isinstance(prop_value, (str, int, float, bool)):
-                        placeholder = f"${{Value.{prop_name}}}"
-                        if placeholder in sub_val:
-                            sub_val = sub_val.replace(
-                                placeholder, str(prop_value)
-                            )
-                            modified = True
+                if isdict(value_obj):
+                    for prop_name, prop_value in value_obj.items():
+                        if isinstance(prop_value, (str, int, float, bool)):
+                            placeholder = f"${{Value.{prop_name}}}"
+                            if placeholder in sub_val:
+                                sub_val = sub_val.replace(
+                                    placeholder, str(prop_value)
+                                )
+                                modified = True
 
                 if modified:
                     if not is_sub_needed(sub_val):
@@ -635,13 +645,10 @@ def resolve_value_refs(v):
                         v.d[SUB] = sub_val
 
         # Handle GetAtt to $Value.X
-        elif isinstance(v.d, dict) and GETATT in v.d:
-            getatt = v.d[GETATT]
-            if isinstance(getatt, list) and len(getatt) >= 2:
-                if getatt[0] == "$Value" and isinstance(value_obj, dict):
-                    prop_name = getatt[1]
-                    if prop_name in value_obj:
-                        v.p[v.k] = value_obj[prop_name]
+        else:
+            value = _get_value_from_getatt_reference(v.d, value_obj)
+            if value is not None:
+                v.p[v.k] = value
 
     Visitor(copied_module[PROPERTIES]).visit(resolve_value_refs)
 

awscli/customizations/cloudformation/modules/process.py

@@ -64,7 +64,9 @@
     is_sub_needed,
 )
 from awscli.customizations.cloudformation.modules.visitor import Visitor
-from awscli.customizations.cloudformation import yamlhelper
+from awscli.customizations.cloudformation.yamlhelper import (
+    yaml_parse,
+)
 from awscli.customizations.cloudformation.modules.names import (
     RESOURCES,
     METADATA,
@@ -368,6 +370,9 @@ def __init__(self, template, module_config, parent_module, s3_client=None):
         # {"Name": Length}
         self.foreach_modules = {}
 
+        # Similar to foreeach_modules, for Resources with a ForEach
+        self.resource_identifiers = {}
+
     def __str__(self):
         "Print out a string with module details for logs"
         return (
@@ -385,7 +390,7 @@ def process(self):
         content, lines = read_source(self.source, self.s3_client)
         self.lines = lines
 
-        module_dict = yamlhelper.yaml_parse(content)
+        module_dict = yaml_parse(content)
         self.original_module_dict = copy.deepcopy(module_dict)
         return self.process_content(module_dict)
 
@@ -422,6 +427,7 @@ def process_content(self, module_dict):
             # The module may only have sub modules in the Modules section
             self.resources = {}
         else:
+            # Store the resources first
             self.resources = module_dict[RESOURCES]
 
         if PARAMETERS in module_dict:
@@ -467,6 +473,15 @@ def process_content(self, module_dict):
         self.validate_overrides()
 
         if not self.invoked:
+            # Process ForEach in resources before emitting them to the parent template
+            if RESOURCES in module_dict:
+                self.resource_identifiers = process_foreach(
+                    module_dict, self.parent_module
+                )
+
+                # Update resources after ForEach processing
+                self.resources = module_dict[RESOURCES]
+
             # Process resources and put them into the parent template
             for logical_id, resource in self.resources.items():
                 self.process_resource(logical_id, resource)
@@ -690,8 +705,6 @@ def resolve_output_getatt(self, v, d, n):
         else:
             foreach_modules = self.foreach_modules
 
-        print("resolve_output_getatt foreach_modules:", foreach_modules)
-
         name = v[0]
         prop_name = v[1]
 
@@ -750,6 +763,57 @@ def resolve_output_getatt(self, v, d, n):
         if reffed_prop is None:
             return False
 
+        # Handle the case where the output value is a GetAtt to a resource with ForEach
+        if isinstance(reffed_prop, dict) and GETATT in reffed_prop:
+            getatt_ref = reffed_prop[GETATT]
+            if isinstance(getatt_ref, list) and len(getatt_ref) >= 2:
+                resource_name = getatt_ref[0]
+                property_path = getatt_ref[1]
+
+                # Check if this is a reference to a resource with ForEach using [*] syntax
+                if "[*]" in resource_name:
+                    base_name = resource_name.split("[")[0]
+                    # Check if we have resource identifiers from ForEach processing
+                    if (
+                        hasattr(self, "resource_identifiers")
+                        and base_name in self.resource_identifiers
+                    ):
+                        # Create a list of GetAtt references to each copied resource
+                        resource_list = []
+                        for i, _ in enumerate(
+                            self.resource_identifiers[base_name]
+                        ):
+                            resource_list.append(
+                                {
+                                    GETATT: [
+                                        f"{self.name}{base_name}{i}",
+                                        property_path,
+                                    ]
+                                }
+                            )
+                        d[n] = resource_list
+                        return True
+
+                # Check if this is a reference to a specific instance using [identifier] syntax
+                elif "[" in resource_name and "]" in resource_name:
+                    base_name = resource_name.split("[")[0]
+                    identifier = resource_name.split("[")[1].split("]")[0]
+
+                    if (
+                        hasattr(self, "resource_identifiers")
+                        and base_name in self.resource_identifiers
+                    ):
+                        keys = self.resource_identifiers[base_name]
+                        if identifier in keys:
+                            index = keys.index(identifier)
+                            d[n] = {
+                                GETATT: [
+                                    f"{self.name}{base_name}{index}",
+                                    property_path,
+                                ]
+                            }
+                            return True
+
         if isinstance(reffed_prop, list):
             for i, r in enumerate(reffed_prop):
                 self.replace_reffed_prop(r, reffed_prop, i)

tests/unit/customizations/cloudformation/modules/module-resource-foreach-module.yaml

@@ -13,7 +13,7 @@ Resources:
       BucketName: !Sub "${BucketPrefix}-bucket-${Identifier}"
       Tags:
         - Key: Environment
-          Value: !Ref Identifier
+          Value: !Sub ${Identifier}
 
 Outputs:
   BucketArns: