Feature/multiple templates

docs/dev/designs/multiple_templates.md

@@ -0,0 +1,25 @@
+# Ability to set separate versions of templates for BG domain parts
+
+## Problem statement
+
+EnvGene needs to be able to use different template artifacts for rendering BGD
+namespaces: common, peer, origin.
+
+## Proposed solutions
+
+### A
+
+Render all 3 templates and replace bgd namespaces in common template with
+namespaces from 2 templates
+
+### B
+
+Render template/environment descriptors for all template artifact.
+
+bg_domain object is now rendered before namespaces.
+
+During namespaces rendering check bg_domain and if namespace is origin or peer,
+use namespace template and template_override value from respective template/environment descriptor.
+
+During processing paramsets check namespace role and use paramsets from respective
+templates.

python/envgene/envgenehelper/business_helper.py

@@ -1,8 +1,10 @@
 from dataclasses import dataclass, field
+from enum import Enum
 import re
 from os import getenv
 from pathlib import Path
 from typing import overload
+from functools import cache
 
 from ruyaml import CommentedMap
 
@@ -360,15 +362,35 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir):
     else:
         return ""
 
+class NamespaceRole(Enum):
+    COMMON = 1
+    ORIGIN = 2
+    PEER = 3
+
+def get_namespace_role(ns_name: str, bgd_object: dict | None = None) -> NamespaceRole:
+    if not bgd_object:
+        bgd_object = get_bgd_object()
+    if not bgd_object:
+        return NamespaceRole.COMMON
+    if bgd_object['originNamespace']['name'] == ns_name:
+        return NamespaceRole.ORIGIN
+    if bgd_object['peerNamespace']['name'] == ns_name:
+        return NamespaceRole.PEER
+    return NamespaceRole.COMMON
+
 @dataclass
 class NamespaceFile:
     path: Path
     name: str = field(init=False)
+    postfix: str = field(init=False)
     definition_path: Path = field(init=False)
+    role: NamespaceRole = field(init=False)
 
     def __post_init__(self):
         self.definition_path = self.path.joinpath('namespace.yml')
         self.name = openYaml(self.definition_path)['name']
+        self.postfix = self.path.name
+        self.role = get_namespace_role(self.name)
 
 def get_namespaces_path(env_dir: Path | None = None) -> Path:
     env_dir = env_dir or get_current_env_dir_from_env_vars()
@@ -391,6 +413,7 @@ def get_bgd_path() -> Path:
     logger.debug(bgd_path)
     return bgd_path
 
+@cache
 def get_bgd_object() -> CommentedMap:
     bgd_path = get_bgd_path()
     bgd_object = openYaml(bgd_path, allow_default=True)

scripts/build_env/build_env.py

@@ -10,6 +10,7 @@
 from schema_validation import checkEnvSpecificParametersBySchema
 from cloud_passport import process_cloud_passport
 from pathlib import Path
+from envgenehelper.business_helper import get_namespace_role, get_namespaces
 
 # const
 GENERATED_HEADER = "The contents of this file is generated from template artifact: %s.\nContents will be overwritten by next generation.\nPlease modify this contents only for development purposes or as workaround."
@@ -460,6 +461,30 @@ def getTemplateNameFromNamespacePath(namespacePath):
     path = pathlib.Path(namespacePath)
     return path.parent.name
 
+def create_role_specific_paramset_map(base_paramset_map: dict, parameters_dir: str, role: str) -> dict:
+    role_dir_suffix = f'from_template_{role}_ns'
+    role_specific_dir = os.path.join(parameters_dir, role_dir_suffix)
+
+    role_paramset_map = createParamsetsMap(role_specific_dir)
+
+    if not role_paramset_map:
+        return base_paramset_map
+
+    merged_map = copy.deepcopy(base_paramset_map)
+
+    for paramset_name, entries in role_paramset_map.items():
+        if paramset_name not in merged_map:
+            merged_map[paramset_name] = entries
+            continue
+        # Keep paramsets from_instance and replace template paramsets
+        filtered_base_entries = [
+            e for e in merged_map[paramset_name]
+            if 'from_template' not in e['filePath'] or role_dir_suffix in e['filePath']
+        ]
+        merged_map[paramset_name] = filtered_base_entries + entries
+
+    logger.info(f"Created {role}-specific paramset map with {len(role_paramset_map)} role-specific paramsets")
+    return merged_map
 
 def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, resource_profiles_dir,
               env_specific_resource_profile_map, all_instances_dir, render_context):
@@ -482,7 +507,7 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res
     # pathes
     tenantTemplatePath = env_dir + "/tenant.yml"
     cloudTemlatePath = env_dir + "/cloud.yml"
-    namespaceTemplates = find_namespaces(env_dir)
+    namespaces = get_namespaces()
     # env specific parameters map - will be filled with env specific parameters during template processing
     env_specific_parameters_map = {}
     env_specific_parameters_map["namespaces"] = {}
@@ -522,21 +547,34 @@ def build_env(env_name, env_instances_dir, parameters_dir, env_template_dir, res
 
     # process namespaces
     template_namespace_names = []
+
+    # Create role-specific paramset maps if needed
+    origin_paramset_map = create_role_specific_paramset_map(paramset_map, parameters_dir, 'origin')
+    peer_paramset_map = create_role_specific_paramset_map(paramset_map, parameters_dir, 'peer')
+
     # iterate through namespace definitions and create namespace parameters
-    for templatePath in namespaceTemplates:
-        logger.info(f"Processing namespace: {templatePath}")
-        templateName = getTemplateNameFromNamespacePath(templatePath)
-        template_namespace_names.append(templateName)
-        initParametersStructure(env_specific_parameters_map["namespaces"], templateName)
+    for ns in namespaces:
+        logger.info(f"Processing namespace: {ns.definition_path}")
+        template_namespace_names.append(ns.postfix)
+        initParametersStructure(env_specific_parameters_map['namespaces'], ns.postfix)
+
+        if ns.role == NamespaceRole.ORIGIN:
+            ns_paramset_map = origin_paramset_map
+        elif ns.role == NamespaceRole.PEER:
+            ns_paramset_map = peer_paramset_map
+        else:
+            ns_paramset_map = paramset_map
+
         processTemplate(
-            templatePath,
-            templateName,
+            ns.definition_path,
+            ns.postfix,
             env_instances_dir,
             namespace_schema,
-            paramset_map,
-            env_specific_parameters_map["namespaces"][templateName],
+            ns_paramset_map,
+            env_specific_parameters_map['namespaces'][ns.postfix],
             resource_profiles_map=needed_resource_profiles_map,
-            header_text=generated_header_text)
+            header_text=generated_header_text,
+        )
 
     logger.info(f"EnvSpecific parameters are: \n{dump_as_yaml_format(env_specific_parameters_map)}")
     checkEnvSpecificParametersBySchema(env_dir, env_specific_parameters_map, template_namespace_names)

scripts/build_env/env_template/process_env_template.py

@@ -3,33 +3,34 @@
 import tempfile
 from pathlib import Path
 
+from pkg_resources import functools
+
 from artifact_searcher import artifact
 from artifact_searcher.utils.models import FileExtension, Application, Credentials, Registry
 from envgenehelper import getEnvDefinition, fetch_cred_value
 from envgenehelper import openYaml, find_all_yaml_files_by_stem, getenv_with_error, logger
 from envgenehelper import unpack_archive, get_cred_config
 
-artifact_dest = f"{tempfile.gettempdir()}/artifact.zip"
 build_env_path = "/build_env"
+ORIGIN_NS_TEMPLATE_PATH = "/build_env_origin_ns"
+PEER_NS_TEMPLATE_PATH = "/build_env_peer_ns"
 
 
-def parse_artifact_appver(env_definition: dict) -> [str, str]:
-    artifact_appver = env_definition['envTemplate'].get('artifact', '')
+def parse_artifact_appver(artifact_appver: str) -> list[str]:
     logger.info(f"Environment template artifact version: {artifact_appver}")
     return artifact_appver.split(':')
 
-
-def load_artifact_definition(name: str) -> Application:
-    base_dir = getenv_with_error('CI_PROJECT_DIR')
+# TODO not actually safe, do something else
+@functools.lru_cache(maxsize=2, typed=False)
+def load_artifact_definition(name: str, base_dir: str) -> Application:
     path_pattern = os.path.join(base_dir, 'configuration', 'artifact_definitions', name)
     path = next(iter(find_all_yaml_files_by_stem(path_pattern)), None)
     if not path:
         raise FileNotFoundError(f"No artifact definition file found for {name} with .yaml or .yml extension")
     return Application.model_validate(openYaml(path))
 
 
-def get_registry_creds(registry: Registry) -> Credentials:
-    cred_config = get_cred_config()
+def get_registry_creds(registry: Registry, cred_config: dict) -> Credentials:
     cred_id = registry.credentials_id
     if cred_id:
         username = cred_config[cred_id]['data'].get('username')
@@ -54,10 +55,10 @@ def extract_snapshot_version(url: str, snapshot_version: str) -> str:
 
 
 # logic downloading template by artifact definition
-def download_artifact_new_logic(env_definition: dict) -> str:
-    app_name, app_version = parse_artifact_appver(env_definition)
-    app_def = load_artifact_definition(app_name)
-    cred = get_registry_creds(app_def.registry)
+def download_artifact_new_logic(artifact_appver: str, target_path: str, base_dir: str, cred_config: dict) -> str:
+    app_name, app_version = parse_artifact_appver(artifact_appver)
+    app_def = load_artifact_definition(app_name, base_dir)
+    cred = get_registry_creds(app_def.registry, cred_config)
     template_url = None
 
     resolved_version = app_version
@@ -89,13 +90,24 @@ def download_artifact_new_logic(env_definition: dict) -> str:
     if not template_url:
         raise ValueError(f"artifact not found group_id={group_id}, artifact_id={artifact_id}, version={version}")
     logger.info(f"Environment template url has been resolved: {template_url}")
-    artifact.download(template_url, artifact_dest, cred)
-    unpack_archive(artifact_dest, build_env_path)
+
+    # Use a unique temporary file for each download to avoid conflicts
+    with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
+        artifact_dest = tmp_file.name
+
+    try:
+        artifact.download(template_url, artifact_dest, cred)
+        unpack_archive(artifact_dest, target_path)
+    finally:
+        # Clean up the temporary file
+        if os.path.exists(artifact_dest):
+            os.unlink(artifact_dest)
+
     return resolved_version
 
 
 # logic downloading template by exact coordinates and repo, deprecated
-def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str:
+def download_artifact_old_logic(env_definition: dict, project_dir: str, cred_config: dict) -> str:
     template_artifact = env_definition['envTemplate']['templateArtifact']
     artifact_info = template_artifact['artifact']
 
@@ -111,7 +123,6 @@ def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str:
     repo_url = registry.get(repo_type)
     dd_repo_url = registry.get(dd_repo_type)
 
-    cred_config = get_cred_config()
     repository_username = fetch_cred_value(registry.get("username"), cred_config)
     repository_password = fetch_cred_value(registry.get("password"), cred_config)
     cred = Credentials(username=repository_username, password=repository_password)
@@ -137,21 +148,80 @@ def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str:
             resolved_version = extract_snapshot_version(template_url, dd_version)
 
     logger.info(f"Environment template url has been resolved: {template_url}")
-    artifact.download(template_url, artifact_dest, cred)
-    unpack_archive(artifact_dest, build_env_path)
+
+    # Use a unique temporary file for each download to avoid conflicts
+    with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
+        artifact_dest = tmp_file.name
+
+    try:
+        artifact.download(template_url, artifact_dest, cred)
+        unpack_archive(artifact_dest, build_env_path)
+    finally:
+        # Clean up the temporary file
+        if os.path.exists(artifact_dest):
+            os.unlink(artifact_dest)
+
     return resolved_version
 
 
-def process_env_template() -> str:
+async def download_artifact_new_logic_async(artifact_appver: str, target_path: str, base_dir: str, cred_config: dict) -> str:
+    """Async wrapper for download_artifact_new_logic to enable concurrent downloads"""
+    return await asyncio.to_thread(download_artifact_new_logic, artifact_appver, target_path, base_dir, cred_config)
+
+
+async def download_artifact_old_logic_async(env_definition: dict, project_dir: str, cred_config: dict) -> str:
+    """Async wrapper for download_artifact_old_logic to enable concurrent downloads"""
+    return await asyncio.to_thread(download_artifact_old_logic, env_definition, project_dir, cred_config)
+
+
+def process_env_template() -> tuple[str, str | None, str | None]:
     project_dir = getenv_with_error("CI_PROJECT_DIR")
     cluster = getenv_with_error("CLUSTER_NAME")
     environment = getenv_with_error("ENVIRONMENT_NAME")
     env_dir = Path(f"{project_dir}/environments/{cluster}/{environment}")
     env_definition = getEnvDefinition(env_dir)
+    env_template = env_definition.get('envTemplate', {})
 
-    if 'artifact' in env_definition.get('envTemplate', {}):
-        logger.info("Use template downloading new logic")
-        return download_artifact_new_logic(env_definition)
-    else:
-        logger.info("Use template downloading old logic")
-        return download_artifact_old_logic(env_definition, project_dir)
+    cred_config = get_cred_config()
+
+    bg_artifacts_key = 'bgNsArtifacts'
+    bg_artifacts = env_template.get(bg_artifacts_key, {})
+
+    async def download_all_templates():
+        if 'artifact' in env_template:
+            logger.info("Use template downloading new logic")
+            main_task = download_artifact_new_logic_async(
+                env_template.get('artifact', ''),
+                build_env_path,
+                project_dir,
+                cred_config
+            )
+        else:
+            logger.info("Use template downloading old logic")
+            main_task = download_artifact_old_logic_async(env_definition, project_dir, cred_config)
+
+        tasks = [main_task]
+        task_labels = ['main']
+
+        bg_artifact_configs = [
+            ('origin', ORIGIN_NS_TEMPLATE_PATH),
+            ('peer', PEER_NS_TEMPLATE_PATH),
+        ]
+
+        for artifact_key, target_path in bg_artifact_configs:
+            artifact_appver = bg_artifacts.get(artifact_key)
+            if artifact_appver:
+                logger.info(f'Try to download template for appver: {artifact_appver}, from {bg_artifacts_key}.{artifact_key}')
+                tasks.append(download_artifact_new_logic_async(artifact_appver, target_path, project_dir, cred_config))
+                task_labels.append(artifact_key)
+
+        results = await asyncio.gather(*tasks)
+
+        result_map = dict(zip(task_labels, results))
+        return (
+            result_map['main'],
+            result_map.get('origin', None),
+            result_map.get('peer', None)
+        )
+
+    return asyncio.run(download_all_templates())