From 9538570d688f6ef83b7d3d4b981f69bdad5644d3 Mon Sep 17 00:00:00 2001 From: GeethaGadde99 Date: Tue, 3 Feb 2026 15:09:49 +0530 Subject: [PATCH 1/4] feat: Enhance Test data for Calculator CLI --- .../pl-01/Namespaces/monitoring/namespace.yml | 4 + .../cleanup/monitoring/parameters.yaml | 24 +++ .../effective-set/cleanup/pg/parameters.yaml | 20 +++ .../MONITORING/values/credentials.yaml | 2 +- .../values/deployment-parameters.yaml | 138 ++++++++++++------ .../values/deployment-parameters.yaml | 50 ++++++- .../environments/cluster-01/pl-01/tenant.yml | 14 +- .../expression/ExpressionLanguage.java | 8 + .../processor/expression/binding/Binding.java | 2 +- 9 files changed, 203 insertions(+), 59 deletions(-) diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml index d794157dc..f00fc37a0 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml @@ -13,6 +13,10 @@ profile: name: "dev_bss_override" baseline: "dev" deployParameters: + server_port: 8080 + app_version: "3.0" + ssl_enabled: true + debug_mode_test: "true" ENVGENE_CONFIG_REF_NAME: "branch_name" ENVGENE_CONFIG_TAG: "No Ref tag" KMS_CERT_IN_BASE64: "${creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml index 89e294958..6393cdd18 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml @@ -71,9 +71,33 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +app_version: '3.0' bss-app-exist: false core: apps: volumes: outputs: capacity: 20Gi +database_config: + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +ssl_enabled: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml index 7edd43101..afa5daa13 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml @@ -69,3 +69,23 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +database_config: + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml index 3baf4a632..c45480d21 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml @@ -62,8 +62,8 @@ global: &id002 TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 - kafka: *id001 graphite-remote-adapter: user-placeholder-123 + kafka: *id001 alertmanager: *id002 blackbox-exporter: *id002 cert-exporter: *id002 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml index e9f387b25..90351cf62 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml @@ -72,13 +72,37 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +app_version: '3.0' bss-app-exist: false -core: &id001 +core: &id002 apps: volumes: outputs: capacity: 20Gi -global: &id002 +database_config: &id003 + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +ssl_enabled: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id004 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: MONITORING ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.system.monitoring_monitoring-operator @@ -153,50 +177,68 @@ global: &id002 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 + api_config: *id001 + app_version: '3.0' bss-app-exist: false - core: *id001 -alertmanager: *id002 -blackbox-exporter: *id002 -cert-exporter: *id002 -cloud-events-exporter: *id002 -cloud-events-reader: *id002 -cloudwatch-exporter: *id002 -common-dashboards: *id002 -configmap-reload: *id002 -configurations-streamer: *id002 -grafana: *id002 -grafana-image-renderer: *id002 -grafana-operator: *id002 -grafana-plugins-init: *id002 -graphite-remote-adapter: *id002 -json-exporter: *id002 -kube-rbac-proxy: *id002 -kube-state-metrics: *id002 -monitoring-operator: *id002 -network-latency-exporter: *id002 -node-exporter: *id002 -oauth2-proxy: *id002 -platform_monitoring_tests: *id002 -prometheus: *id002 -prometheus-adapter: *id002 -prometheus-adapter-converter: *id002 -prometheus-adapter-operator: *id002 -prometheus-config-reloader: *id002 -prometheus-operator: *id002 -promitor-agent-resource-discovery: *id002 -promitor-agent-scraper: *id002 -promxy: *id002 -pushgateway: *id002 -stackdriver-exporter: *id002 -version-exporter: *id002 -victoriametrics-operator: *id002 -vmagent: *id002 -vmalert: *id002 -vmauth: *id002 -vmcleanup: *id002 -vminsert: *id002 -vmoperator: *id002 -vmoperator_config_reloader: *id002 -vmselect: *id002 -vmsingle: *id002 -vmstorage: *id002 + core: *id002 + database_config: *id003 + debug_mode_test: 'true' + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + server_port: 8080 + ssl_enabled: true + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +alertmanager: *id004 +blackbox-exporter: *id004 +cert-exporter: *id004 +cloud-events-exporter: *id004 +cloud-events-reader: *id004 +cloudwatch-exporter: *id004 +common-dashboards: *id004 +configmap-reload: *id004 +configurations-streamer: *id004 +grafana: *id004 +grafana-image-renderer: *id004 +grafana-operator: *id004 +grafana-plugins-init: *id004 +graphite-remote-adapter: *id004 +json-exporter: *id004 +kube-rbac-proxy: *id004 +kube-state-metrics: *id004 +monitoring-operator: *id004 +network-latency-exporter: *id004 +node-exporter: *id004 +oauth2-proxy: *id004 +platform_monitoring_tests: *id004 +prometheus: *id004 +prometheus-adapter: *id004 +prometheus-adapter-converter: *id004 +prometheus-adapter-operator: *id004 +prometheus-config-reloader: *id004 +prometheus-operator: *id004 +promitor-agent-resource-discovery: *id004 +promitor-agent-scraper: *id004 +promxy: *id004 +pushgateway: *id004 +stackdriver-exporter: *id004 +version-exporter: *id004 +victoriametrics-operator: *id004 +vmagent: *id004 +vmalert: *id004 +vmauth: *id004 +vmcleanup: *id004 +vminsert: *id004 +vmoperator: *id004 +vmoperator_config_reloader: *id004 +vmselect: *id004 +vmsingle: *id004 +vmstorage: *id004 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml index 9d4f8d5ca..d1c9a4d9a 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml @@ -70,7 +70,27 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -global: &id001 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +database_config: &id002 + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id003 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: postgres ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.ha.postgres @@ -143,16 +163,30 @@ global: &id001 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 + api_config: *id001 + database_config: *id002 pg_patroni: artifactory.qubership.org:17125/product/prod.platform.ha_pg-patroni:release-2024.2-2.10.0_20240715-123045_pg16 pg_upgrade: artifactory.qubership.org:17125/product/prod.platform.ha_pg-service:release-2024.2-2.10.0_20240716-142315_pg-upgrade pgbackrest_sidecar: artifactory.qubership.org:17263/qubership/pgskipper-pgbackrest-sidecar:release-2024.2-2.10.0 postgres_operator_init: artifactory.qubership.org:17208/product/prod.platform.ha_postgres-operator:2.10.5_20241215-141230_init postgres_operator_tests: artifactory.qubership.org:17263/qubership/pgskipper-operator-tests:release-2024.2-2.10.0 + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 vault_env: artifactory.qubership.org:17175/banzaicloud/vault-env:2.5.1 -patroni-core: *id001 -pg_patroni: *id001 -pg_upgrade: *id001 -pgbackrest_sidecar: *id001 -postgres_operator_init: *id001 -postgres_operator_tests: *id001 -vault_env: *id001 + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +patroni-core: *id003 +pg_patroni: *id003 +pg_upgrade: *id003 +pgbackrest_sidecar: *id003 +postgres_operator_init: *id003 +postgres_operator_tests: *id003 +vault_env: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml index c526990a0..d267792c6 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml @@ -15,4 +15,16 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: - ESCAPE_SEQUENCE: "true" \ No newline at end of file + ESCAPE_SEQUENCE: "true" + api_config: ${database_config} + database_config: + connection: + host: db.example.com + port: 5432 + yaml_template: | + services: + api: + image: api:latest + ports: + - 8080:8080 + rendered_template: ${yaml_template} diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java index cb7a571c0..67c6cec4b 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java @@ -220,6 +220,14 @@ private Parameter processValue(Object value, Map binding, boo String rendered = ""; + + if (strValue.matches("^\\$\\{[^}]+}$")) { + String key = strValue.substring(2, strValue.length() - 1); + Parameter ref = binding.get(key); + if (ref != null) { + return ref; + } + } this.binding.getTypeCollector().clear(); try { rendered = renderStringByJinJava(strValue, binding, escapeDollar); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java index c6c3bec29..959cc4ba6 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java @@ -41,7 +41,7 @@ public class Binding extends HashMap implements Cloneable { String escapeSequence; @Getter private DeployerInputs deployerInputs; - private Map defaultMap; + private Map defaultMap = new HashMap<>(); private final Map calculatedMap = new HashMap<>(); @Getter private String tenant; From 6e2f105f3c86def4f9fb084e138ac5c29645a313 Mon Sep 17 00:00:00 2001 From: GeethaGadde99 Date: Tue, 3 Feb 2026 21:46:05 +0530 Subject: [PATCH 2/4] feat: Enhance Test data for Calculator CLI --- .../values/deployment-parameters.yaml | 50 ++++++++++++++++--- .../effective-set/topology/credentials.yaml | 2 +- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml index c11ace9fc..dcbe130aa 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml @@ -69,7 +69,27 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -global: &id001 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +database_config: &id002 + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id003 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: postgres ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.ha.postgres @@ -141,10 +161,24 @@ global: &id001 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -patroni-core: *id001 -pg_patroni: *id001 -pg_upgrade: *id001 -pgbackrest_sidecar: *id001 -postgres_operator_init: *id001 -postgres_operator_tests: *id001 -vault_env: *id001 + api_config: *id001 + database_config: *id002 + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +patroni-core: *id003 +pg_patroni: *id003 +pg_upgrade: *id003 +pgbackrest_sidecar: *id003 +postgres_operator_init: *id003 +postgres_operator_tests: *id003 +vault_env: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml index 38c4c40a6..0adf9f21c 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml @@ -1,7 +1,7 @@ bg_domain: controllerNamespace: - password: pass-placeholder-123 username: user-placeholder-123 + password: pass-placeholder-123 k8s_tokens: monitoring-origin: token-placeholder-123 pg: token-placeholder-123 From 1a43770fda9abce0b4cc8a81dc66d9829537b3a4 Mon Sep 17 00:00:00 2001 From: Geetha Gadde Date: Wed, 4 Feb 2026 11:02:55 +0530 Subject: [PATCH 3/4] feat: enhance the test data for calculator cli (#986) * fix: change deploy_param processing in case of collision (#957) * chore: Update docker image tags and envgene_version for branch main [skip ci] * docs: Separate the Application/Registry Definitions processing logic into a dedicated `app_reg_def_process` job (#750) * docs: Separate the Application/Registry Definitions processing logic into a dedicated `app_reg_def_process` job * docs: app_reg_def_process update * feat: Add tracebitity comments in effective Set and Clarify the priority of sources for Effective Set (#789) * docs: add Traceability in the Effective Set * docs: minor * docs: minor * docs: Traceability Comments update * docs: minor * docs: minor * docs: update pipeline description (#962) * feat: Separate the Application/Registry Definitions processing logic into a dedicated app_reg_def_process job (#950) * chore: Update docker image tags and envgene_version for branch main [skip ci] * docs: unify-logging (#967) * feat: 850 Deploy postfix handling for BGD cases (#965) * chore: Update docker image tags and envgene_version for branch main [skip ci] * feat: Removed the values from pipeline_vars.env (#968) * chore: Update docker image tags and envgene_version for branch main [skip ci] * feat: Added new separators for ENV_NAMES (#969) * chore: Added separators to Env_Names * chore: Added new delimeters to generate env matrix script * feat: Added APP_REG_DEF_PROCESS job to Envgene Instance Pipeline (Github) (#974) * chore: Update docker image tags and envgene_version for branch main [skip ci] * feat: Implement new pipeline parameter "ENV_INVENTORY_CONTENT" for create/replace inventory, paramsets, credentials, and resource profile overrides (#977) Co-authored-by: popoveugene * chore: Update docker image tags and envgene_version for branch main [skip ci] * fix: validation for inv gen job (#984) * chore: Update docker image tags and envgene_version for branch main [skip ci] * fix: skip env def validation for ENV_INVENTORY_CONTENT * feat: Enhance Test data for Calculator CLI * feat: Enhance Test data for Calculator CLI --------- Co-authored-by: miyamuraga <198181742+miyamuraga@users.noreply.github.com> Co-authored-by: qubership-actions[bot] Co-authored-by: popoveugene <42543333+popoveugene@users.noreply.github.com> Co-authored-by: Dias <120464230+dysmon@users.noreply.github.com> Co-authored-by: Siva Reddy Kunduru <35566000+sivareddyit@users.noreply.github.com> Co-authored-by: Andrei Rudchenko <104736077+andyroode@users.noreply.github.com> Co-authored-by: popoveugene --- .gitignore | 2 +- .../FileDataRepositoryImpl.java | 48 +- .../cli/utils/BomReaderUtilsImplV2.java | 6 +- .../Applications/crm.yml | 0 .../namespace.yml | 4 + .../cluster-01/pl-01/bg_domain.yml | 13 + .../pl-01/effective-set/cleanup/mapping.yaml | 2 +- .../credentials.yaml | 0 .../parameters.yaml | 30 +- .../effective-set/cleanup/pg/parameters.yaml | 29 +- .../effective-set/deployment/mapping.yaml | 2 +- .../values/collision-credentials.yaml | 0 .../collision-deployment-parameters.yaml | 0 .../MONITORING/values/credentials.yaml | 2 +- .../MONITORING/values/deploy-descriptor.yaml | 0 .../values/deployment-parameters.yaml | 150 ++-- .../monitoring-sf/deployment-parameters.yaml | 0 .../collision-deployment-parameters.yaml | 6 - .../values/deployment-parameters.yaml | 74 +- .../pl-01/effective-set/runtime/mapping.yaml | 2 +- .../MONITORING/credentials.yaml | 0 .../MONITORING/parameters.yaml | 0 .../effective-set/topology/credentials.yaml | 7 +- .../effective-set/topology/parameters.yaml | 19 +- .../environments/cluster-01/pl-01/tenant.yml | 14 +- .../expression/ExpressionLanguage.java | 8 + .../processor/expression/binding/Binding.java | 2 +- build_envgene/scripts/git_commit.sh | 61 +- .../scripts/appregdef_render_job.py | 49 + build_pipegene/scripts/env_build_jobs.py | 28 +- build_pipegene/scripts/gitlab_ci.py | 161 ++-- .../scripts/inventory_generation_job.py | 46 +- build_pipegene/scripts/validations.py | 10 +- docs/dev/unify-logging.md | 70 ++ docs/envgene-objects.md | 35 + docs/envgene-pipelines.md | 35 +- docs/features/app-reg-defs.md | 4 +- docs/features/calculator-cli.md | 313 +++++-- docs/features/env-inventory-generation.md | 578 +++++++++++- docs/instance-pipeline-parameters.md | 74 +- .../environment-instance-generation.md | 2 +- .../environment-inventory-generation.md | 839 ++++++++++++++++++ .../.github/configuration/config.env | 1 + .../.github/jobs/build_env_job.sh | 76 -- .../.github/pipeline_vars.env | 11 - .../scripts/create_env_generation_params.sh | 2 +- .../.github/scripts/generate_env_matrix.sh | 5 +- .../.github/workflows/Envgene.yml | 112 ++- .../git-system-follower-package/package.yaml | 2 +- .../git-system-follower-package/package.yaml | 2 +- .../templates/default/cookiecutter.json | 2 +- .../envgene/envgenehelper/business_helper.py | 37 +- python/envgene/envgenehelper/file_helper.py | 9 +- python/envgene/envgenehelper/yaml_helper.py | 44 +- schemas/env-definition.schema.json | 2 +- schemas/env-inventory-content.schema.json | 209 +++++ scripts/build_env/appregdef_render.py | 44 + scripts/build_env/env_inventory_generation.py | 141 ++- .../env_template/process_env_template.py | 67 +- .../env_template/template_testing.py | 10 +- scripts/build_env/main.py | 61 +- scripts/build_env/render_config_env.py | 202 +++-- .../tests/app_reg_defs/test_appregdefs.py | 142 +++ scripts/build_env/tests/base_test.py | 16 + .../tests/env-build/test_render_envs.py | 72 +- .../test_env_inv_generation.py | 114 +++ scripts/utils/pipeline_parameters.py | 3 +- .../env-01/Inventory/env_definition.yml | 35 + .../expected/appdefs/application-1.yaml} | 0 .../templates/appdefs/application-1.yaml.j2 | 7 + .../env-01/Inventory/env_definition.yml | 35 + .../expected/regdefs/registry-1.yaml} | 0 .../templates/regdefs/registry-1.yaml.j2 | 19 + .../env-01/Inventory/env_definition.yml | 35 + .../configuration/appregdef_config.yaml | 4 + .../expected/appdefs/application-1.yaml} | 2 +- .../templates/appdefs/application-1.yaml.j2 | 7 + .../env-01/Inventory/env_definition.yml | 35 + .../configuration/appregdef_config.yaml | 5 + .../expected/regdefs/registry-1.yaml} | 6 +- .../templates/regdefs/registry-1.yaml.j2} | 6 +- .../configuration/appregdef_config.yaml | 4 + .../env-01/Inventory/env_definition.yml | 35 + .../expected/appdefs/application-1.yaml | 7 + .../templates/appdefs/application-1.yaml.j2 | 7 + .../configuration/appregdef_config.yaml | 4 + .../env-01/Inventory/env_definition.yml | 35 + .../configuration/appregdef_config.yaml | 5 + .../expected/appdefs/application-1.yaml | 7 + .../templates/appdefs/application-1.yaml.j2 | 7 + .../env-01/Inventory/env_definition.yml | 35 + .../expected/appdefs/application-1.yaml} | 0 .../expected/appdefs/application-2.yml.j2} | 2 - .../expected/appdefs/application-3.yml | 7 + .../templates/appdefs/application-1.yaml.j2 | 7 + .../templates/appdefs/application-2.yml.j2} | 4 +- .../templates/appdefs/application-3.yml | 7 + .../env-01/Inventory/env_definition.yml | 35 + .../templates/appdefs/application-1.yaml.j2 | 7 + .../env-01/Inventory/env_definition.yml | 35 + .../templates/appdefs/application-1.yaml.j2 | 0 .../env-01/Inventory/env_definition.yml | 35 + .../templates/regdefs/registry-1.yaml.j2 | 18 + .../{origin-app => app-origin}/namespace.yml | 0 .../{peer-app => app-peer}/namespace.yml | 0 .../bgd-env/RegDefs/registry-2.yml | 19 - .../bgd-cluster/bgd-env/bg_domain.yml | 4 +- .../cluster-01/env-01/RegDefs/registry-2.yml | 19 - .../AppDefs/application-2.yml | 9 - .../RegDefs/registry-2.yml | 19 - .../expected/configuration/config.yml | 1 + .../shared-credentials-cluster.yml | 5 + .../credentials/shared-credentials-env.yml | 5 + .../env-01/Inventory/env_definition.yml | 35 + .../parameters/env-specific-bss-env.yml | 5 + .../cloud-specific-profile-env.yml | 15 + .../prod-template-variables-env.yml | 5 + .../parameters/env-specific-bss-cluster.yml | 5 + .../cloud-specific-profile-cluster.yml | 15 + .../prod-template-variables-cluster.yml | 5 + .../credentials/shared-credentials-site.yml | 5 + .../parameters/env-specific-bss-site.yml | 5 + .../cloud-specific-profile-site.yml | 15 + .../prod-template-variables-site.yml | 5 + .../input/content.yml.j2 | 144 +++ .../test_templates/env_templates/bgd.yaml | 6 +- .../env_templates/bgd/bg_domain.yml.j2 | 6 +- 127 files changed, 4128 insertions(+), 834 deletions(-) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/{monitoring => monitoring-origin}/Applications/crm.yml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/{monitoring => monitoring-origin}/namespace.yml (97%) create mode 100644 build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/bg_domain.yml rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/{monitoring => monitoring-origin}/credentials.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/{monitoring => monitoring-origin}/parameters.yaml (81%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/collision-credentials.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/collision-deployment-parameters.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/credentials.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/deploy-descriptor.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/deployment-parameters.yaml (71%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/{monitoring => monitoring-origin}/MONITORING/values/per-service-parameters/monitoring-sf/deployment-parameters.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/{monitoring => monitoring-origin}/MONITORING/credentials.yaml (100%) rename build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/{monitoring => monitoring-origin}/MONITORING/parameters.yaml (100%) create mode 100644 build_pipegene/scripts/appregdef_render_job.py create mode 100644 docs/dev/unify-logging.md create mode 100644 docs/use-cases/environment-inventory-generation.md delete mode 100755 github_workflows/instance-repo-pipeline/.github/jobs/build_env_job.sh create mode 100644 schemas/env-inventory-content.schema.json create mode 100644 scripts/build_env/appregdef_render.py create mode 100644 scripts/build_env/tests/app_reg_defs/test_appregdefs.py create mode 100644 scripts/build_env/tests/base_test.py create mode 100644 scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py create mode 100644 test_data/test_app_reg_defs/TC-001-001/environments/cluster-01/env-01/Inventory/env_definition.yml rename test_data/{test_environments/bgd-cluster/bgd-env/AppDefs/application-1.yml => test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yaml} (100%) create mode 100644 test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-002/environments/cluster-01/env-01/Inventory/env_definition.yml rename test_data/{test_environments/bgd-cluster/bgd-env/RegDefs/registry-1.yml => test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yaml} (100%) create mode 100644 test_data/test_app_reg_defs/TC-001-002/templates/regdefs/registry-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-003/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-003/environments/configuration/appregdef_config.yaml rename test_data/{test_environments/cluster03/rpo-replacement-mode/AppDefs/application-1.yml => test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml} (83%) create mode 100644 test_data/test_app_reg_defs/TC-001-003/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-004/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-004/environments/configuration/appregdef_config.yaml rename test_data/{test_environments/cluster-01/env-01/RegDefs/registry-1.yml => test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml} (78%) rename test_data/{test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-1.yml => test_app_reg_defs/TC-001-004/templates/regdefs/registry-1.yaml.j2} (63%) create mode 100644 test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/configuration/appregdef_config.yaml create mode 100644 test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml create mode 100644 test_data/test_app_reg_defs/TC-001-005/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/configuration/appregdef_config.yaml create mode 100644 test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-006/environments/configuration/appregdef_config.yaml create mode 100644 test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml create mode 100644 test_data/test_app_reg_defs/TC-001-006/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-008/environments/cluster-01/env-01/Inventory/env_definition.yml rename test_data/{test_environments/cluster-01/env-01/AppDefs/application-1.yml => test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml} (100%) rename test_data/{test_environments/cluster-01/env-01/AppDefs/application-2.yml => test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2} (77%) create mode 100644 test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml create mode 100644 test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-1.yaml.j2 rename test_data/{test_environments/bgd-cluster/bgd-env/AppDefs/application-2.yml => test_app_reg_defs/TC-001-008/templates/appdefs/application-2.yml.j2} (60%) create mode 100644 test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-3.yml create mode 100644 test_data/test_app_reg_defs/TC-001-010/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-010/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-011/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-011/templates/appdefs/application-1.yaml.j2 create mode 100644 test_data/test_app_reg_defs/TC-001-012/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_app_reg_defs/TC-001-012/templates/regdefs/registry-1.yaml.j2 rename test_data/test_environments/bgd-cluster/bgd-env/Namespaces/{origin-app => app-origin}/namespace.yml (100%) rename test_data/test_environments/bgd-cluster/bgd-env/Namespaces/{peer-app => app-peer}/namespace.yml (100%) delete mode 100644 test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-2.yml delete mode 100644 test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml delete mode 100644 test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-2.yml delete mode 100644 test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-2.yml create mode 100644 test_data/test_inventory_generation/expected/configuration/config.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/credentials/shared-credentials-cluster.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/credentials/shared-credentials-env.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/env_definition.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/parameters/env-specific-bss-env.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/resource_profiles/cloud-specific-profile-env.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/env-01/shared_template_variables/prod-template-variables-env.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/parameters/env-specific-bss-cluster.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/resource_profiles/cloud-specific-profile-cluster.yml create mode 100644 test_data/test_inventory_generation/expected/environments/cluster-01/shared_template_variables/prod-template-variables-cluster.yml create mode 100644 test_data/test_inventory_generation/expected/environments/credentials/shared-credentials-site.yml create mode 100644 test_data/test_inventory_generation/expected/environments/parameters/env-specific-bss-site.yml create mode 100644 test_data/test_inventory_generation/expected/environments/resource_profiles/cloud-specific-profile-site.yml create mode 100644 test_data/test_inventory_generation/expected/environments/shared_template_variables/prod-template-variables-site.yml create mode 100644 test_data/test_inventory_generation/input/content.yml.j2 diff --git a/.gitignore b/.gitignore index 6644552db..82f61d8db 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ venv .idea /.project /environments -/tmp +tmp/ /env-builder/deployers __pycache__ .vscode diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java index 66cc5be65..f741b9133 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/repository/implementation/FileDataRepositoryImpl.java @@ -112,6 +112,7 @@ public void prepareProcessingEnv() { loadConsumerData(); traverseSourceDirectory(nsWithAppsFromSD, appsToProcess); populateEnvironments(); + correctDeployPostfix(); fileSystemUtils.createEffectiveSetFolder(inputData.getSolutionBomDTO()); } catch (Exception e) { throw new FileParseException("Error preparing data due to " + e.getMessage()); @@ -119,6 +120,14 @@ public void prepareProcessingEnv() { } + private void correctDeployPostfix() { + List applicationDTOList = inputData.getSolutionBomDTO().map(SolutionBomDTO::getApplications) + .orElseGet(Collections::emptyList); + applicationDTOList.parallelStream().forEach(app -> { + app.setNamespace(getNamespaceName(app.getNamespace())); + }); + } + private void loadConsumerData() { Map consumerDTOMap = new LinkedHashMap<>(); if (CollectionUtils.isNotEmpty(sharedData.getPcsspPaths())) { @@ -227,7 +236,8 @@ private void traverseSourceDirectory(Map> nsWithAppsFromSD, public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { if (dir.getParent().getFileName().toString().equals(GenericConstants.NS_FOLDER)) { String namespace = dir.getFileName().toString(); - if (!nsWithAppsFromSD.containsKey(namespace)) { + String correctedNamespace = getCorrectedNamespace(dir.getFileName().toString()); + if (!nsWithAppsFromSD.containsKey(correctedNamespace)) { Path namespaceYaml = dir.resolve("namespace.yml"); if (Files.exists(namespaceYaml)) { NamespaceDTO namespaceDTO = fileDataConverter.parseInputFile(NamespaceDTO.class, namespaceYaml.toFile()); @@ -275,6 +285,15 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { inputData.setCloudDTO(inputData.getCloudDTO().toBuilder().applications(cloudApps).build()); } + private String getCorrectedNamespace(String namespace) { + if (namespace.endsWith("-origin")) { + return namespace.substring(0, namespace.indexOf("-origin")); + } else if (namespace.endsWith("-peer")) { + return namespace.substring(0, namespace.indexOf("-peer")); + } + return namespace; + } + private void handleNamespaceYamlFile(Path file, Map> clusterMap) { Path parent = file.getParent(); String name = file.getFileName().toString().split("\\.")[0]; @@ -329,7 +348,7 @@ private void handleYamlFile(Path file, Map profilesMap, CompositeStructureDTO compositeStructureDTO = fileDataConverter.parseInputFile(CompositeStructureDTO.class, file.toFile()); inputData.setCompositeStructureDTO(compositeStructureDTO); break; - case "bg-domain": + case "bg_domain": BgDomainEntityDTO bgDomainEntityDTO = fileDataConverter.parseInputFile(BgDomainEntityDTO.class, file.toFile()); inputData.setBgDomainEntityDTO(bgDomainEntityDTO); break; @@ -362,8 +381,9 @@ private void processApplicationFiles(Path file, Path parent, Map new ArrayList<>()).add(applicationLinkDTO); } } else { @@ -392,7 +412,7 @@ private SBApplicationDTO getSbApplicationDTO(Map> nsWithApp String namespace = applicationDTO.getDeployPostfix(); String appName = applicationDTO.getVersion().split(":")[0]; String appVersion = applicationDTO.getVersion().replace(":", "-"); - String appFileRef = String.format("%s/%s", sharedData.getSbomsPath().get(), appVersion +".sbom.json"); + String appFileRef = String.format("%s/%s", sharedData.getSbomsPath().get(), appVersion + ".sbom.json"); SBApplicationDTO dto = SBApplicationDTO.builder() .appName(appName) .appVersion(appVersion) @@ -404,6 +424,26 @@ private SBApplicationDTO getSbApplicationDTO(Map> nsWithApp return dto; } + private String getNamespaceName(String deployPostFix) { + if (inputData.getBgDomainEntityDTO() != null) { + NamespaceDTO namespaceDTO = + inputData.getNamespaceDTOMap().getOrDefault(deployPostFix, + inputData.getNamespaceDTOMap().getOrDefault(deployPostFix + "-origin", + inputData.getNamespaceDTOMap().get(deployPostFix + "-peer"))); + if (namespaceDTO != null) { + String originalNamespace = namespaceDTO.getName(); + BgDomainEntityDTO.NamespaceDTO originNamespaceDTO = inputData.getBgDomainEntityDTO().getOriginNamespace(); + BgDomainEntityDTO.NamespaceDTO peerNamespaceDTO = inputData.getBgDomainEntityDTO().getPeerNamespace(); + if (originalNamespace.equalsIgnoreCase(originNamespaceDTO.getName())) { + return deployPostFix + "-origin"; + } else if (originalNamespace.equalsIgnoreCase(peerNamespaceDTO.getName())) { + return deployPostFix + "-peer"; + } + } + } + return deployPostFix; + } + private void loadRegistryData() { Optional registryPath = sharedData.getRegistryPath(); if (registryPath.isPresent()) { diff --git a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java index cdd94c0d4..b8977971c 100644 --- a/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java +++ b/build_effective_set_generator/effective-set-generator/src/main/java/org/qubership/cloud/devops/cli/utils/BomReaderUtilsImplV2.java @@ -311,7 +311,7 @@ private void processImageServiceComponent(EntitiesMap entitiesMap, Component com String dockerTag = getPropertyValue(component, "full_image_name", null, true, entity); serviceParams.put("DOCKER_TAG", dockerTag); serviceParams.put("IMAGE_REPOSITORY", getImageRepository(dockerTag)); - addImageParameters(component, entitiesMap.getDeployParams()); + addImageParameters(component, entitiesMap.getDeployParams(), entitiesMap.getServiceMap().keySet()); if (CollectionUtils.isNotEmpty(component.getComponents())) { for (Component subComponent : component.getComponents()) { @@ -329,10 +329,10 @@ private void processImageServiceComponent(EntitiesMap entitiesMap, Component com perServiceMap.put(component.getName(), serviceParams); } - private void addImageParameters(Component component, Map serviceParams) { + private void addImageParameters(Component component, Map serviceParams, Set serviceNames) { if (component.getMimeType().equalsIgnoreCase(APPLICATION_OCTET_STREAM)) { String key = getPropertyValue(component, "deploy_param", null, false, component.getName()); - if (StringUtils.isNotEmpty(key)) { + if (StringUtils.isNotEmpty(key) && !serviceNames.contains(key)) { String value = getPropertyValue(component, "full_image_name", null, false, component.getName()); serviceParams.put(key, value); } diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/Applications/crm.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/Applications/crm.yml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/Applications/crm.yml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/Applications/crm.yml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml similarity index 97% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml index d794157dc..f00fc37a0 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring/namespace.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml @@ -13,6 +13,10 @@ profile: name: "dev_bss_override" baseline: "dev" deployParameters: + server_port: 8080 + app_version: "3.0" + ssl_enabled: true + debug_mode_test: "true" ENVGENE_CONFIG_REF_NAME: "branch_name" ENVGENE_CONFIG_TAG: "No Ref tag" KMS_CERT_IN_BASE64: "${creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/bg_domain.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/bg_domain.yml new file mode 100644 index 000000000..79aa4897f --- /dev/null +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/bg_domain.yml @@ -0,0 +1,13 @@ +name: "env-1-bg-domain" +type: bgdomain +originNamespace: + name: "pl-01-monitoring" + type: namespace +peerNamespace: + name: "pl-01-pg-test" + type: namespace +controllerNamespace: + name: "env-1-bg-controller" + type: namespace + credentials: creds-get-password-cred + url: https://controller-env-1-controller.qubership.org diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/mapping.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/mapping.yaml index 9c0dd7b88..c54cade22 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/mapping.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/mapping.yaml @@ -1,2 +1,2 @@ -pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/cleanup/monitoring +pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin pl-01-pg: /environments/cluster-01/pl-01/effective-set/cleanup/pg diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/credentials.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/credentials.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml similarity index 81% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml index 89e294958..66dd2aa24 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml @@ -15,6 +15,7 @@ CONSUL_ADMIN_TOKEN: token-placeholder-123 CONSUL_ENABLED: 'true' CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 +CONTROLLER_NAMESPACE: env-1-bg-controller CUSTOM_HOST: cluster-01.qubership.org DBAAS_AGGREGATOR_ADDRESS: https://dbaas.cluster-01.qubership.org DBAAS_AGGREGATOR_PASSWORD: pass-placeholder-123 @@ -53,11 +54,12 @@ ORIGIN_NAMESPACE: pl-01-monitoring OVERRIDE_PARAMETER: template-value PARAM_2: value-2 PARAM_6: value-6 +PEER_NAMESPACE: pl-01-pg-test PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org -PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org +PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org -PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org +PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org STATIC_CACHE_SERVICE_ROUTE_HOST: static-cache-service-pl-01-monitoring.cluster-01.qubership.org STORAGE_PROVIDER: s3 @@ -71,9 +73,33 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +app_version: '3.0' bss-app-exist: false core: apps: volumes: outputs: capacity: 20Gi +database_config: + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +ssl_enabled: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml index 7edd43101..bfdf1880c 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/pg/parameters.yaml @@ -47,14 +47,13 @@ MAVEN_REPO_URL: https://artifactory.qubership.org MONITORING_ENABLED: 'true' NAMESPACE: pl-01-pg OPENSHIFT_SERVER: https://api.cluster-01.qubership.org:6443 -ORIGIN_NAMESPACE: pl-01-pg PARAM_2: value-2 PARAM_6: value-6 -PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org -PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org +PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org +PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false -PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org -PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org +PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org +PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org SITE_LEVEL_PARAM_1: site-level-value-1 STATIC_CACHE_SERVICE_ROUTE_HOST: static-cache-service-pl-01-pg.cluster-01.qubership.org @@ -69,3 +68,23 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: + connection: + host: db.example.com + port: 5432 +database_config: + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/mapping.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/mapping.yaml index a83a8a19c..1c50b129a 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/mapping.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/mapping.yaml @@ -1,2 +1,2 @@ -pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/deployment/monitoring +pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin pl-01-pg: /environments/cluster-01/pl-01/effective-set/deployment/pg diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/collision-credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/collision-credentials.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/collision-credentials.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/collision-credentials.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/collision-deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/collision-deployment-parameters.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/collision-deployment-parameters.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/collision-deployment-parameters.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml index 3baf4a632..c45480d21 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/credentials.yaml @@ -62,8 +62,8 @@ global: &id002 TEST_ENVGENE_CREDS_GET_VAULT_SECRET_ID: envgeneNullValue TEST_SHARED_CREDS: user-placeholder-123 TEST_SHARED_CREDS_ACTIVATOR: pass-placeholder-123 - kafka: *id001 graphite-remote-adapter: user-placeholder-123 + kafka: *id001 alertmanager: *id002 blackbox-exporter: *id002 cert-exporter: *id002 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deploy-descriptor.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deploy-descriptor.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deploy-descriptor.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml similarity index 71% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml index e9f387b25..66fd83c5d 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml @@ -20,6 +20,7 @@ CMDB_URL: https://cluster-01.qubership.org CONSUL_ENABLED: 'true' CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 +CONTROLLER_NAMESPACE: env-1-bg-controller CUSTOM_HOST: cluster-01.qubership.org DBAAS_AGGREGATOR_ADDRESS: https://dbaas.cluster-01.qubership.org DBAAS_ENABLED: true @@ -53,11 +54,12 @@ ORIGIN_NAMESPACE: pl-01-monitoring OVERRIDE_PARAMETER: template-value PARAM_2: value-2 PARAM_6: value-6 +PEER_NAMESPACE: pl-01-pg-test PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org -PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org +PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org -PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org +PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org SSL_SECRET: defaultsslcertificate STATIC_CACHE_SERVICE_ROUTE_HOST: static-cache-service-pl-01-monitoring.cluster-01.qubership.org @@ -72,13 +74,37 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +app_version: '3.0' bss-app-exist: false -core: &id001 +core: &id002 apps: volumes: outputs: capacity: 20Gi -global: &id002 +database_config: &id003 + connection: + host: db.example.com + port: 5432 +debug_mode_test: 'true' +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +server_port: 8080 +ssl_enabled: true +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id004 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: MONITORING ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.system.monitoring_monitoring-operator @@ -101,6 +127,7 @@ global: &id002 CONSUL_ENABLED: 'true' CONSUL_PUBLIC_URL: http://consul.consul:8080 CONSUL_URL: http://consul.consul:8080 + CONTROLLER_NAMESPACE: env-1-bg-controller CUSTOM_HOST: cluster-01.qubership.org DBAAS_AGGREGATOR_ADDRESS: https://dbaas.cluster-01.qubership.org DBAAS_ENABLED: true @@ -134,11 +161,12 @@ global: &id002 OVERRIDE_PARAMETER: template-value PARAM_2: value-2 PARAM_6: value-6 + PEER_NAMESPACE: pl-01-pg-test PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org - PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org + PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org - PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org + PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org SSL_SECRET: defaultsslcertificate STATIC_CACHE_SERVICE_ROUTE_HOST: static-cache-service-pl-01-monitoring.cluster-01.qubership.org @@ -153,50 +181,68 @@ global: &id002 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 + api_config: *id001 + app_version: '3.0' bss-app-exist: false - core: *id001 -alertmanager: *id002 -blackbox-exporter: *id002 -cert-exporter: *id002 -cloud-events-exporter: *id002 -cloud-events-reader: *id002 -cloudwatch-exporter: *id002 -common-dashboards: *id002 -configmap-reload: *id002 -configurations-streamer: *id002 -grafana: *id002 -grafana-image-renderer: *id002 -grafana-operator: *id002 -grafana-plugins-init: *id002 -graphite-remote-adapter: *id002 -json-exporter: *id002 -kube-rbac-proxy: *id002 -kube-state-metrics: *id002 -monitoring-operator: *id002 -network-latency-exporter: *id002 -node-exporter: *id002 -oauth2-proxy: *id002 -platform_monitoring_tests: *id002 -prometheus: *id002 -prometheus-adapter: *id002 -prometheus-adapter-converter: *id002 -prometheus-adapter-operator: *id002 -prometheus-config-reloader: *id002 -prometheus-operator: *id002 -promitor-agent-resource-discovery: *id002 -promitor-agent-scraper: *id002 -promxy: *id002 -pushgateway: *id002 -stackdriver-exporter: *id002 -version-exporter: *id002 -victoriametrics-operator: *id002 -vmagent: *id002 -vmalert: *id002 -vmauth: *id002 -vmcleanup: *id002 -vminsert: *id002 -vmoperator: *id002 -vmoperator_config_reloader: *id002 -vmselect: *id002 -vmsingle: *id002 -vmstorage: *id002 + core: *id002 + database_config: *id003 + debug_mode_test: 'true' + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + server_port: 8080 + ssl_enabled: true + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +alertmanager: *id004 +blackbox-exporter: *id004 +cert-exporter: *id004 +cloud-events-exporter: *id004 +cloud-events-reader: *id004 +cloudwatch-exporter: *id004 +common-dashboards: *id004 +configmap-reload: *id004 +configurations-streamer: *id004 +grafana: *id004 +grafana-image-renderer: *id004 +grafana-operator: *id004 +grafana-plugins-init: *id004 +graphite-remote-adapter: *id004 +json-exporter: *id004 +kube-rbac-proxy: *id004 +kube-state-metrics: *id004 +monitoring-operator: *id004 +network-latency-exporter: *id004 +node-exporter: *id004 +oauth2-proxy: *id004 +platform_monitoring_tests: *id004 +prometheus: *id004 +prometheus-adapter: *id004 +prometheus-adapter-converter: *id004 +prometheus-adapter-operator: *id004 +prometheus-config-reloader: *id004 +prometheus-operator: *id004 +promitor-agent-resource-discovery: *id004 +promitor-agent-scraper: *id004 +promxy: *id004 +pushgateway: *id004 +stackdriver-exporter: *id004 +version-exporter: *id004 +victoriametrics-operator: *id004 +vmagent: *id004 +vmalert: *id004 +vmauth: *id004 +vmcleanup: *id004 +vminsert: *id004 +vmoperator: *id004 +vmoperator_config_reloader: *id004 +vmselect: *id004 +vmsingle: *id004 +vmstorage: *id004 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/per-service-parameters/monitoring-sf/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/per-service-parameters/monitoring-sf/deployment-parameters.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring/MONITORING/values/per-service-parameters/monitoring-sf/deployment-parameters.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/per-service-parameters/monitoring-sf/deployment-parameters.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/collision-deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/collision-deployment-parameters.yaml index 65d96727c..e69de29bb 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/collision-deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/collision-deployment-parameters.yaml @@ -1,6 +0,0 @@ -pg_patroni: artifactory.qubership.org:17125/product/prod.platform.ha_pg-patroni:release-2024.2-2.10.0_20240715-123045_pg16 -pg_upgrade: artifactory.qubership.org:17125/product/prod.platform.ha_pg-service:release-2024.2-2.10.0_20240716-142315_pg-upgrade -pgbackrest_sidecar: artifactory.qubership.org:17263/qubership/pgskipper-pgbackrest-sidecar:release-2024.2-2.10.0 -postgres_operator_init: artifactory.qubership.org:17208/product/prod.platform.ha_postgres-operator:2.10.5_20241215-141230_init -postgres_operator_tests: artifactory.qubership.org:17263/qubership/pgskipper-operator-tests:release-2024.2-2.10.0 -vault_env: artifactory.qubership.org:17175/banzaicloud/vault-env:2.5.1 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml index 9d4f8d5ca..dcbe130aa 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/pg/postgres/values/deployment-parameters.yaml @@ -47,14 +47,13 @@ MAVEN_REPO_URL: https://artifactory.qubership.org MONITORING_ENABLED: 'true' NAMESPACE: pl-01-pg OPENSHIFT_SERVER: https://api.cluster-01.qubership.org:6443 -ORIGIN_NAMESPACE: pl-01-pg PARAM_2: value-2 PARAM_6: value-6 -PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org -PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org +PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org +PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false -PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org -PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org +PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org +PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org SITE_LEVEL_PARAM_1: site-level-value-1 SSL_SECRET: defaultsslcertificate @@ -70,7 +69,27 @@ TRACING_HOST: tracing-agent TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 -global: &id001 +api_config: &id001 + connection: + host: db.example.com + port: 5432 +database_config: &id002 + connection: + host: db.example.com + port: 5432 +rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +global: &id003 API_DBAAS_ADDRESS: http://dbaas.dbaas:8080 APPLICATION_NAME: postgres ARTIFACT_DESCRIPTOR_ARTIFACT_ID: prod.platform.ha.postgres @@ -120,14 +139,13 @@ global: &id001 MONITORING_ENABLED: 'true' NAMESPACE: pl-01-pg OPENSHIFT_SERVER: https://api.cluster-01.qubership.org:6443 - ORIGIN_NAMESPACE: pl-01-pg PARAM_2: value-2 PARAM_6: value-6 - PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org - PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-pl-01-pg.cluster-01.qubership.org + PRIVATE_GATEWAY_URL: https://private-gateway-pl-01-monitoring.cluster-01.qubership.org + PRIVATE_IDENTITY_PROVIDER_URL: https://private-gateway-env-1-bg-domain.cluster-01.qubership.org PRODUCTION_MODE: false - PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org - PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-pl-01-pg.cluster-01.qubership.org + PUBLIC_GATEWAY_URL: https://public-gateway-pl-01-monitoring.cluster-01.qubership.org + PUBLIC_IDENTITY_PROVIDER_URL: https://public-gateway-env-1-bg-domain.cluster-01.qubership.org SERVER_HOSTNAME: cluster-01.qubership.org SITE_LEVEL_PARAM_1: site-level-value-1 SSL_SECRET: defaultsslcertificate @@ -143,16 +161,24 @@ global: &id001 TRACING_UI_URL: https://cluster-01.qubership.org ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 - pg_patroni: artifactory.qubership.org:17125/product/prod.platform.ha_pg-patroni:release-2024.2-2.10.0_20240715-123045_pg16 - pg_upgrade: artifactory.qubership.org:17125/product/prod.platform.ha_pg-service:release-2024.2-2.10.0_20240716-142315_pg-upgrade - pgbackrest_sidecar: artifactory.qubership.org:17263/qubership/pgskipper-pgbackrest-sidecar:release-2024.2-2.10.0 - postgres_operator_init: artifactory.qubership.org:17208/product/prod.platform.ha_postgres-operator:2.10.5_20241215-141230_init - postgres_operator_tests: artifactory.qubership.org:17263/qubership/pgskipper-operator-tests:release-2024.2-2.10.0 - vault_env: artifactory.qubership.org:17175/banzaicloud/vault-env:2.5.1 -patroni-core: *id001 -pg_patroni: *id001 -pg_upgrade: *id001 -pgbackrest_sidecar: *id001 -postgres_operator_init: *id001 -postgres_operator_tests: *id001 -vault_env: *id001 + api_config: *id001 + database_config: *id002 + rendered_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 + yaml_template: |- + services: + api: + image: api:latest + ports: + - 8080:8080 +patroni-core: *id003 +pg_patroni: *id003 +pg_upgrade: *id003 +pgbackrest_sidecar: *id003 +postgres_operator_init: *id003 +postgres_operator_tests: *id003 +vault_env: *id003 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/mapping.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/mapping.yaml index e2e8fbda7..35c897966 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/mapping.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/mapping.yaml @@ -1,2 +1,2 @@ -pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/runtime/monitoring +pl-01-monitoring: /environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin pl-01-pg: /environments/cluster-01/pl-01/effective-set/runtime/pg diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring/MONITORING/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring/MONITORING/credentials.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/credentials.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring/MONITORING/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml similarity index 100% rename from build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring/MONITORING/parameters.yaml rename to build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/runtime/monitoring-origin/MONITORING/parameters.yaml diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml index 77c32045b..0adf9f21c 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/credentials.yaml @@ -1,6 +1,9 @@ -bg_domain: {} +bg_domain: + controllerNamespace: + username: user-placeholder-123 + password: pass-placeholder-123 k8s_tokens: + monitoring-origin: token-placeholder-123 pg: token-placeholder-123 pl-01-pg: token-placeholder-123 pl-01-monitoring: token-placeholder-123 - monitoring: token-placeholder-123 diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/parameters.yaml index 47e3b1274..e453ed902 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/topology/parameters.yaml @@ -1,4 +1,19 @@ -bg_domain: {} +bg_domain: + name: env-1-bg-domain + type: bgdomain + originNamespace: + name: pl-01-monitoring + type: namespace + credentials: null + peerNamespace: + name: pl-01-pg-test + type: namespace + credentials: null + controllerNamespace: + name: env-1-bg-controller + type: namespace + credentials: creds-get-password-cred + url: https://controller-env-1-controller.qubership.org cluster: api_port: '6443' api_url: api.cluster-01.qubership.org @@ -9,6 +24,6 @@ environments: cluster-01/pl-01: namespaces: pl-01-monitoring: - deployPostfix: monitoring + deployPostfix: monitoring-origin pl-01-pg: deployPostfix: pg diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml index c526990a0..d267792c6 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/tenant.yml @@ -15,4 +15,16 @@ globalE2EParameters: mergeTenantsAndE2EParameters: false environmentParameters: {} deployParameters: - ESCAPE_SEQUENCE: "true" \ No newline at end of file + ESCAPE_SEQUENCE: "true" + api_config: ${database_config} + database_config: + connection: + host: db.example.com + port: 5432 + yaml_template: | + services: + api: + image: api:latest + ports: + - 8080:8080 + rendered_template: ${yaml_template} diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java index cb7a571c0..67c6cec4b 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/ExpressionLanguage.java @@ -220,6 +220,14 @@ private Parameter processValue(Object value, Map binding, boo String rendered = ""; + + if (strValue.matches("^\\$\\{[^}]+}$")) { + String key = strValue.substring(2, strValue.length() - 1); + Parameter ref = binding.get(key); + if (ref != null) { + return ref; + } + } this.binding.getTypeCollector().clear(); try { rendered = renderStringByJinJava(strValue, binding, escapeDollar); diff --git a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java index c6c3bec29..959cc4ba6 100644 --- a/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java +++ b/build_effective_set_generator/parameters-processor/src/main/java/org/qubership/cloud/parameters/processor/expression/binding/Binding.java @@ -41,7 +41,7 @@ public class Binding extends HashMap implements Cloneable { String escapeSequence; @Getter private DeployerInputs deployerInputs; - private Map defaultMap; + private Map defaultMap = new HashMap<>(); private final Map calculatedMap = new HashMap<>(); @Getter private String tenant; diff --git a/build_envgene/scripts/git_commit.sh b/build_envgene/scripts/git_commit.sh index 910ab3e4e..314a75330 100755 --- a/build_envgene/scripts/git_commit.sh +++ b/build_envgene/scripts/git_commit.sh @@ -67,7 +67,7 @@ echo "Commit message: ${message}" echo "Moving env environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME} artifacts to temporary location" mkdir -p /tmp/artifact_environments/${CLUSTER_NAME} -if [ "${COMMIT_ENV}" = "true" ]; then +if [ "${COMMIT_ENV}" = "true" ] && [ -d "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" ]; then cp -r environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME} /tmp/artifact_environments/${CLUSTER_NAME}/ fi @@ -97,6 +97,31 @@ if [ -e gitlab-ci/prefix_build ]; then cp -r templates /tmp fi +echo "Saving cluster and site shared folders" +mkdir -p /tmp/artifact_shared + +if [ -d environments/${CLUSTER_NAME} ]; then + for dir in parameters credentials resource_profiles shared_template_variables; do + SRC="environments/${CLUSTER_NAME}/$dir" + if [ -d "$SRC" ]; then + echo "Saving $SRC" + mkdir -p "/tmp/artifact_shared/environments/${CLUSTER_NAME}" + cp -r "$SRC" "/tmp/artifact_shared/environments/${CLUSTER_NAME}/" + fi + done +fi + +if [ -d environments ]; then + for dir in parameters credentials resource_profiles shared_template_variables; do + SRC="environments/$dir" + if [ -d "$SRC" ]; then + echo "Saving $SRC" + mkdir -p "/tmp/artifact_shared/environments" + cp -r "$SRC" "/tmp/artifact_shared/environments/" + fi + done +fi + #Copying cred files modified as part of cred rotation job. CREDS_FILE="environments/credfilestoupdate.yml" if [ -f "$CREDS_FILE" ]; then @@ -153,11 +178,39 @@ echo "Pulling contents from GIT (branch: ${REF_NAME})" git pull origin "${REF_NAME}" # moving back environments folder and committing + +echo "Restoring cluster and site shared folders" + +for dir in parameters credentials resource_profiles shared_template_variables; do + if [ -d /tmp/artifact_shared/environments/$dir ]; then + rm -rf environments/$dir + mkdir -p environments/$dir + cp -r /tmp/artifact_shared/environments/$dir/. environments/$dir/ + fi +done + +for dir in parameters credentials resource_profiles shared_template_variables; do + if [ -d /tmp/artifact_shared/environments/${CLUSTER_NAME}/$dir ]; then + rm -rf environments/${CLUSTER_NAME}/$dir + mkdir -p environments/${CLUSTER_NAME}/$dir + cp -r /tmp/artifact_shared/environments/${CLUSTER_NAME}/$dir/. environments/${CLUSTER_NAME}/$dir/ + fi +done + + echo "Restoring environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" + if [ "${COMMIT_ENV}" = "true" ]; then - rm -rf "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" - mkdir -p "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" - cp -r /tmp/artifact_environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}/. "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}/" + if [ -d "/tmp/artifact_environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" ]; then + rm -rf "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" + mkdir -p "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" + cp -r "/tmp/artifact_environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}/." "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}/" + else + if [ -d "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" ]; then + echo "Folder environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME} no longer exists, deleting from repo" + rm -rf "environments/${CLUSTER_NAME}/${ENVIRONMENT_NAME}" + fi + fi fi if [ -e /tmp/artifact_environments/${CLUSTER_NAME}/cloud-passport ]; then diff --git a/build_pipegene/scripts/appregdef_render_job.py b/build_pipegene/scripts/appregdef_render_job.py new file mode 100644 index 000000000..1537b7df1 --- /dev/null +++ b/build_pipegene/scripts/appregdef_render_job.py @@ -0,0 +1,49 @@ +from gcip import WhenStatement + +from envgenehelper import logger +from pipeline_helper import job_instance + + +def prepare_appregdef_render_job(pipeline, is_template_test, env_template_version, full_env, environment_name, + cluster_name, group_id, artifact_id, artifact_url, tags): + logger.info(f'Prepare appregdef render job for {full_env}') + script = [ + '/module/scripts/handle_certs.sh', + ] + + if env_template_version and not is_template_test: + script.append('python3 /build_env/scripts/build_env/env_template/set_template_version.py') + + script.append('cd /build_env; python3 /build_env/scripts/build_env/appregdef_render.py') + + appregdef_render_params = { + "name": f'app_reg_def_render.{full_env}', + "image": '${envgen_image}', + "stage": 'app_reg_def_render', + "script": script + } + + appregdef_render_vars = { + "ENV_NAME": full_env, + "FULL_ENV_NAME": full_env, + "CLUSTER_NAME": cluster_name, + "ENVIRONMENT_NAME": environment_name, + "ENV_TEMPLATE_TEST": "true" if is_template_test else "false", + "ENV_TEMPLATE_VERSION": env_template_version, + "INSTANCES_DIR": "${CI_PROJECT_DIR}/environments", + "GROUP_ID": group_id, + "ARTIFACT_ID": artifact_id, + "ARTIFACT_URL": artifact_url, + "GITLAB_RUNNER_TAG_NAME": tags, + } + + appregdef_render_job = job_instance(params=appregdef_render_params, vars=appregdef_render_vars) + + appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/{full_env}") + appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/configuration") + appregdef_render_job.artifacts.add_paths("${CI_PROJECT_DIR}/tmp") + appregdef_render_job.artifacts.when = WhenStatement.ALWAYS + + pipeline.add_children(appregdef_render_job) + + return appregdef_render_job \ No newline at end of file diff --git a/build_pipegene/scripts/env_build_jobs.py b/build_pipegene/scripts/env_build_jobs.py index 724a96011..3dbe4f66e 100644 --- a/build_pipegene/scripts/env_build_jobs.py +++ b/build_pipegene/scripts/env_build_jobs.py @@ -3,42 +3,24 @@ from pipeline_helper import job_instance -def prepare_env_build_job(pipeline, is_template_test, env_template_version, full_env, enviroment_name, cluster_name, - group_id, artifact_id, artifact_url, tags): +def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, cluster_name, group_id, artifact_id, tags): logger.info(f'prepare env_build job for {full_env}') - # prepare script + script = [ '/module/scripts/handle_certs.sh', ] - # adding update template version - if env_template_version and env_template_version != "" and not is_template_test: - script.append('python3 /build_env/scripts/build_env/env_template/set_template_version.py') - script.append('cd /build_env; python3 /build_env/scripts/build_env/main.py') if is_template_test: script.append('env_name=$(cat "$CI_PROJECT_DIR/set_variable.txt")') script.append( 'sed -i "s|\\\"envgeneNullValue\\\"|\\\"test_value\\\"|g" "$CI_PROJECT_DIR/environments/$env_name/Credentials/credentials.yml"') - else: - script.append("export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')") - - script.extend([ - 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', - 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' - ]) - # add after script - after_script = [ - 'mkdir -p "$CI_PROJECT_DIR/tmp"', - 'cp -r /build_env/tmp/* $CI_PROJECT_DIR/tmp' - ] - # + env_build_params = { "name": f'env_builder.{full_env}', "image": '${envgen_image}', "stage": 'env_builder', - "script": script, - "after_script": after_script + "script": script } env_build_vars = { @@ -46,7 +28,6 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full "FULL_ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, "ENVIRONMENT_NAME": enviroment_name, - "ENV_TEMPLATE_VERSION": env_template_version, "GROUP_ID": group_id, "ARTIFACT_ID": artifact_id, "ENV_TEMPLATE_TEST": "true" if is_template_test else "false", @@ -56,7 +37,6 @@ def prepare_env_build_job(pipeline, is_template_test, env_template_version, full "envgen_debug": "true", "module_config_default": "/module/templates/defaults.yaml", "GITLAB_RUNNER_TAG_NAME": tags, - "ARTIFACT_URL": artifact_url } env_build_job = job_instance(params=env_build_params, vars=env_build_vars) diff --git a/build_pipegene/scripts/gitlab_ci.py b/build_pipegene/scripts/gitlab_ci.py index cbb4d5dae..5bd7290ca 100644 --- a/build_pipegene/scripts/gitlab_ci.py +++ b/build_pipegene/scripts/gitlab_ci.py @@ -1,34 +1,34 @@ import os from os import listdir +from envgenehelper import logger, get_cluster_name_from_full_name, get_environment_name_from_full_name, parse_env_names from envgenehelper.plugin_engine import PluginEngine -from envgenehelper import logger, get_cluster_name_from_full_name, get_environment_name_from_full_name, getEnvDefinition, get_env_instances_dir from gcip import JobFilter, Pipeline -import pipeline_helper -from pipeline_helper import get_gav_coordinates_from_build, find_predecessor_job -from passport_jobs import prepare_trigger_passport_job, prepare_passport_job +import pipeline_helper +from appregdef_render_job import prepare_appregdef_render_job +from bg_manage_job import prepare_bg_manage_job +from credential_rotation_job import prepare_credential_rotation_job from env_build_jobs import prepare_env_build_job, prepare_generate_effective_set_job, prepare_git_commit_job from inventory_generation_job import prepare_inventory_generation_job, is_inventory_generation_needed -from credential_rotation_job import prepare_credential_rotation_job -from bg_manage_job import prepare_bg_manage_job +from passport_jobs import prepare_trigger_passport_job, prepare_passport_job +from pipeline_helper import get_gav_coordinates_from_build, find_predecessor_job -project_dir = os.getenv('CI_PROJECT_DIR') or os.getenv('GITHUB_WORKSPACE') +PROJECT_DIR = os.getenv('CI_PROJECT_DIR') or os.getenv('GITHUB_WORKSPACE') +IS_GITLAB = bool(os.getenv('CI_PROJECT_DIR')) and not bool(os.getenv('GITHUB_ACTIONS')) +IS_GITHUB = bool(os.getenv('GITHUB_WORKSPACE')) or bool(os.getenv('GITHUB_ACTIONS')) -is_gitlab = bool(os.getenv('CI_PROJECT_DIR')) and not bool(os.getenv('GITHUB_ACTIONS')) -is_github = bool(os.getenv('GITHUB_WORKSPACE')) or bool(os.getenv('GITHUB_ACTIONS')) +logger.info(f"Detected environment - GitLab: {IS_GITLAB}, GitHub: {IS_GITHUB}") -logger.info(f"Detected environment - GitLab: {is_gitlab}, GitHub: {is_github}") -def build_pipeline(params: dict): - # if we are in template testing during template build - tags=params['GITLAB_RUNNER_TAG_NAME'] +def build_pipeline(params: dict) -> None: + tags = params['GITLAB_RUNNER_TAG_NAME'] artifact_url = None if params['IS_TEMPLATE_TEST']: - logger.info("We are generating jobs in template test mode.") + logger.info("Generating jobs in template test mode.") artifact_url = os.getenv("artifact_url") - templates_dir = f"{project_dir}/templates/env_templates" + templates_dir = f"{PROJECT_DIR}/templates/env_templates" # getting build artifact build_artifact = get_gav_coordinates_from_build() group_id = build_artifact["group_id"] @@ -39,7 +39,7 @@ def build_pipeline(params: dict): os.path.splitext(f)[0] for f in listdir(templates_dir) if os.path.isfile(os.path.join(templates_dir, f)) - and (f.endswith(".yaml") or f.endswith(".yml")) + and (f.endswith(".yaml") or f.endswith(".yml")) ] params['ENV_NAMES'] = "\n".join(templateFiles) else: @@ -54,75 +54,96 @@ def build_pipeline(params: dict): per_env_plugin_engine = PluginEngine(plugins_dir='/module/scripts/pipegene_plugins/per_env') - for env in params['ENV_NAMES'].split("\n"): - logger.info(f'----------------start processing for {env}---------------------') - ci_project_dir = project_dir + env_names = parse_env_names(params['ENV_NAMES']) + if len(env_names) > 1 and is_inventory_generation_needed(params['IS_TEMPLATE_TEST'], params): + raise ValueError( + f"Generating Inventories for multiple Environments in single pipeline is not supported. " + f"ENV_NAMES: {env_names}" + ) + + for full_env_name in env_names: + logger.info(f'----------------start processing for {full_env_name}---------------------') if params['IS_TEMPLATE_TEST']: - cluster_name = "" - environment_name = env - env_definition = {} + env_template_art_vers = params["ENV_TEMPLATE_VERSION"] + env_template_vers_split = env_template_art_vers.split(':')[1].replace('.', '_') + env_template_version_normalized = f"{env_template_vers_split.replace('-', '_')}" + + project_name = os.getenv("CI_PROJECT_NAME") + cluster_name = f"template_testing_{project_name}_{full_env_name}" + environment_name = f"{cluster_name}_{env_template_version_normalized}" else: - cluster_name = get_cluster_name_from_full_name(env) - environment_name = get_environment_name_from_full_name(env) - if params['ENV_INVENTORY_INIT']: - env_definition = None - else: - env_definition = getEnvDefinition(get_env_instances_dir(environment_name, cluster_name, f"{ci_project_dir}/environments")) - - # bg_manage_job -> - # trigger_passport_job -> - # get_passport_job -> - # env_inventory_generation_job -> - # env_build_job -> - # generate_effective_set_job -> - # git_commit_job (commit) -> - job_sequence = ["bg_manage_job", "trigger_passport_job", "get_passport_job", "env_inventory_generation_job", - "credential_rotation_job", "env_build_job", "generate_effective_set_job", "git_commit_job"] - - if params.get('BG_MANAGE', None) != True: - logger.info(f'Preparing of bg_manage job for environment {env} is skipped.') + cluster_name = get_cluster_name_from_full_name(full_env_name) + environment_name = get_environment_name_from_full_name(full_env_name) + + job_sequence = [ + "bg_manage_job", + "trigger_passport_job", + "get_passport_job", + "env_inventory_generation_job", + "credential_rotation_job", + "appregdef_render_job", + "env_build_job", + "generate_effective_set_job", + "git_commit_job" + ] + + if not params.get('BG_MANAGE', None): + logger.info(f'Preparing of bg_manage job for environment {full_env_name} is skipped.') else: - jobs_map['bg_manage_job'] = prepare_bg_manage_job(pipeline, env, tags) + jobs_map['bg_manage_job'] = prepare_bg_manage_job(pipeline, full_env_name, tags) # get passport job if it is not already added for cluster if params['GET_PASSPORT'] and cluster_name not in get_passport_jobs: - jobs_map["trigger_passport_job"] = prepare_trigger_passport_job(pipeline, env) - jobs_map["get_passport_job"] = prepare_passport_job(pipeline, env, environment_name, cluster_name, tags) + jobs_map["trigger_passport_job"] = prepare_trigger_passport_job(pipeline, full_env_name) + jobs_map["get_passport_job"] = prepare_passport_job(pipeline, full_env_name, + environment_name, cluster_name, tags) get_passport_jobs[cluster_name] = True else: - logger.info(f"Generation of cloud passport for environment '{env}' is skipped") + logger.info(f"Generation of cloud passport for environment '{full_env_name}' is skipped") if is_inventory_generation_needed(params['IS_TEMPLATE_TEST'], params): - jobs_map["env_inventory_generation_job"] = prepare_inventory_generation_job(pipeline, env, environment_name, cluster_name, params, tags) + jobs_map["env_inventory_generation_job"] = prepare_inventory_generation_job(pipeline, full_env_name, + environment_name, cluster_name, + params, tags) else: - logger.info(f'Preparing of env inventory generation job for {env} is skipped because we are in template test mode.') + logger.info( + f'Preparing of full_env_name inventory generation job for {full_env_name} ' + f'is skipped because we are in template test mode.') credential_rotation_job = None if params['CRED_ROTATION_PAYLOAD']: - credential_rotation_job = prepare_credential_rotation_job(pipeline, env, environment_name, cluster_name, tags) + credential_rotation_job = prepare_credential_rotation_job(pipeline, full_env_name, environment_name, + cluster_name, tags) jobs_map["credential_rotation_job"] = credential_rotation_job else: - logger.info(f'Credential rotation job for {env} is skipped because CRED_ROTATION_PAYLOAD is empty.') + logger.info( + f'Credential rotation job for {full_env_name} is skipped because CRED_ROTATION_PAYLOAD is empty.') + + if params['ENV_BUILD']: + jobs_map["appregdef_render_job"] = prepare_appregdef_render_job(pipeline, params['IS_TEMPLATE_TEST'], + params['ENV_TEMPLATE_VERSION'], + full_env_name, + environment_name, cluster_name, group_id, + artifact_id, artifact_url, tags) + else: + logger.info(f'Preparing of appregdef_render_job {full_env_name} is skipped.') + if params['ENV_BUILD']: - if env_definition is None: - try: - env_definition = getEnvDefinition(get_env_instances_dir(environment_name, cluster_name, f"{ci_project_dir}/environments")) - except ReferenceError: - pass - - # env_builder job - jobs_map["env_build_job"] = prepare_env_build_job(pipeline, params['IS_TEMPLATE_TEST'], params['ENV_TEMPLATE_VERSION'], env, environment_name, cluster_name, group_id, artifact_id, artifact_url, tags) + jobs_map["env_build_job"] = prepare_env_build_job(pipeline, params['IS_TEMPLATE_TEST'], full_env_name, + environment_name, cluster_name, group_id, artifact_id, + tags) else: - logger.info(f'Preparing of env_build job for {env} is skipped.') + logger.info(f'Preparing of env_build job for {full_env_name} is skipped.') - # generate_effective_set job if params['GENERATE_EFFECTIVE_SET']: - jobs_map["generate_effective_set_job"] = prepare_generate_effective_set_job(pipeline, environment_name, cluster_name, tags) + jobs_map["generate_effective_set_job"] = prepare_generate_effective_set_job(pipeline, environment_name, + cluster_name, tags) else: logger.info(f'Preparing of generate_effective_set job for {cluster_name}/{environment_name} is skipped.') - jobs_requiring_git_commit = ["env_build_job", "generate_effective_set_job", "env_inventory_generation_job", "credential_rotation_job", "bg_manage_job"] + jobs_requiring_git_commit = ["appregdef_render_job", "env_build_job", "generate_effective_set_job", + "env_inventory_generation_job", "credential_rotation_job", "bg_manage_job"] plugin_params = params plugin_params['jobs_map'] = jobs_map @@ -130,14 +151,16 @@ def build_pipeline(params: dict): plugin_params['jobs_requiring_git_commit'] = jobs_requiring_git_commit plugin_params['env_name'] = environment_name plugin_params['cluster_name'] = cluster_name - plugin_params['full_env'] = env + plugin_params['full_env'] = full_env_name per_env_plugin_engine.run(params=plugin_params, pipeline=pipeline, pipeline_helper=pipeline_helper) - ## git_commit job - if any(job in jobs_map for job in plugin_params['jobs_requiring_git_commit']) and not params['IS_TEMPLATE_TEST']: - jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, env, environment_name, cluster_name, params['DEPLOYMENT_SESSION_ID'], tags, credential_rotation_job) + if (any(job in jobs_map for job in plugin_params['jobs_requiring_git_commit']) and + not params['IS_TEMPLATE_TEST']): + jobs_map["git_commit_job"] = prepare_git_commit_job(pipeline, full_env_name, environment_name, cluster_name, + params['DEPLOYMENT_SESSION_ID'], tags, + credential_rotation_job) else: - logger.info(f'Preparing of git commit job for {env} is skipped.') + logger.info(f'Preparing of git commit job for {full_env_name} is skipped.') for job in job_sequence: if job not in jobs_map.keys(): @@ -149,16 +172,14 @@ def build_pipeline(params: dict): sorted_pipeline.add_children(job_instance) job_instance.add_needs(*find_predecessor_job(job, jobs_map, job_sequence)) - logger.info(f'----------------end processing for {env}---------------------') + logger.info(f'----------------end processing for {full_env_name}---------------------') # check out repo only once in the first job of the generated pipeline, later jobs get it through artifacts from each other # purpose: avoid later jobs restoring files that were removed by previous jobs, so git commit job can commit those deletions - for job in sorted_pipeline.find_jobs(JobFilter()): # gets all jobs in pipeline + for job in sorted_pipeline.find_jobs(JobFilter()): # gets all jobs in pipeline job.artifacts.add_paths('./') is_first_job = job.needs is None or len(job.needs) == 0 if not is_first_job: job.add_variables(GIT_CHECKOUT="false") - sorted_pipeline.write_yaml() - diff --git a/build_pipegene/scripts/inventory_generation_job.py b/build_pipegene/scripts/inventory_generation_job.py index 57867e693..8d59703ae 100644 --- a/build_pipegene/scripts/inventory_generation_job.py +++ b/build_pipegene/scripts/inventory_generation_job.py @@ -1,25 +1,38 @@ -from gcip import WhenStatement +import warnings + from envgenehelper import logger +from gcip import WhenStatement + from pipeline_helper import job_instance -import json -def is_inventory_generation_needed(is_template_test, inv_gen_params): +def is_inventory_generation_needed(is_template_test, inventory_params): if is_template_test: return False - if inv_gen_params['ENV_INVENTORY_INIT'] == 'true': - return True - params_processed_by_inv_gen = ['ENV_SPECIFIC_PARAMETERS', 'ENV_TEMPLATE_NAME'] - for param in params_processed_by_inv_gen: - if inv_gen_params[param] not in (None, ''): - return True - return False + + env_inventory_init = inventory_params.get('ENV_INVENTORY_INIT') == 'true' + env_specific_parameters = inventory_params.get('ENV_SPECIFIC_PARAMS') + env_template_name = inventory_params.get('ENV_TEMPLATE_NAME') + env_inventory_content = inventory_params.get('ENV_INVENTORY_CONTENT') + + if env_inventory_content and (env_inventory_init or env_specific_parameters or env_template_name): + warnings.warn( + "ENV_INVENTORY_INIT, ENV_SPECIFIC_PARAMS, and ENV_TEMPLATE_NAME are deprecated", + DeprecationWarning + ) + raise ValueError( + "ENV_INVENTORY_CONTENT cannot be used together with " + "ENV_INVENTORY_INIT, ENV_SPECIFIC_PARAMS, or ENV_TEMPLATE_NAME" + ) + + return env_inventory_content or env_inventory_init or bool(env_specific_parameters) or bool(env_template_name) -def prepare_inventory_generation_job(pipeline, full_env, environment_name, cluster_name, env_generation_params, tags): - logger.info(f"prepare env_generation job for {full_env}") +def prepare_inventory_generation_job(pipeline, full_env_name, environment_name, cluster_name, env_generation_params, + tags): + logger.info(f"prepare env_generation job for {full_env_name}") params = { - "name": f"env_inventory_generation.{full_env}", + "name": f"env_inventory_generation.{full_env_name}", "image": "${envgen_image}", "stage": "env_inventory_generation", "script": [ @@ -30,6 +43,7 @@ def prepare_inventory_generation_job(pipeline, full_env, environment_name, clust vars = { "ENV_NAME": environment_name, "CLUSTER_NAME": cluster_name, + "FULL_ENV_NAME": full_env_name, "envgen_image": "$envgen_image", "envgen_args": " -vv", "envgen_debug": "true", @@ -37,11 +51,11 @@ def prepare_inventory_generation_job(pipeline, full_env, environment_name, clust "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", "module_ansible_cfg": "/module/ansible/ansible.cfg", "module_config_default": "/module/templates/defaults.yaml", - "ENV_GENERATION_PARAMS": json.dumps(env_generation_params, ensure_ascii=False, indent=2), - "GITLAB_RUNNER_TAG_NAME": tags + "GITLAB_RUNNER_TAG_NAME": tags, + **env_generation_params } job = job_instance(params=params, vars=vars) - job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/" + full_env) + job.artifacts.add_paths("${CI_PROJECT_DIR}/environments/") job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(job) return job diff --git a/build_pipegene/scripts/validations.py b/build_pipegene/scripts/validations.py index e6da51b69..582616bb5 100644 --- a/build_pipegene/scripts/validations.py +++ b/build_pipegene/scripts/validations.py @@ -16,6 +16,7 @@ def validate_pipeline(params: dict): params["GET_PASSPORT"], params["ENV_BUILD"], params["ENV_INVENTORY_INIT"], + params["ENV_INVENTORY_CONTENT"] ) def basic_checks(env_names): @@ -40,7 +41,7 @@ def template_test_checks(): if errorFound: raise ReferenceError("Execution is aborted as validation is not successful. See logs above.") -def real_execution_checks(env_names, get_passport, env_build, env_inventory_init): +def real_execution_checks(env_names, get_passport, env_build, env_inventory_init, env_inventory_content): for env in env_names.split("\n"): # now we are using only complex environment names that contain both cluster_name and environment_name if env.count('/') != 1: @@ -50,11 +51,12 @@ def real_execution_checks(env_names, get_passport, env_build, env_inventory_init cluster_name = get_cluster_name_from_full_name(env) environment_name = get_environment_name_from_full_name(env) # checks - check_environment(environment_name, cluster_name, get_passport, env_build, env_inventory_init) + check_environment(environment_name, cluster_name, get_passport, env_build, env_inventory_init, + env_inventory_content) check_passport_params(get_passport) -def check_environment(environment_name, cluster_name, get_passport, env_build, env_inventory_init): - if env_inventory_init == "true": +def check_environment(environment_name, cluster_name, get_passport, env_build, env_inventory_init, env_inventory_content): + if env_inventory_init == "true" or env_inventory_content: return schemas_dir = getenv("JSON_SCHEMAS_DIR", "/module/schemas") all_environments_dir = f"{project_dir}/environments" diff --git a/docs/dev/unify-logging.md b/docs/dev/unify-logging.md new file mode 100644 index 000000000..a450df88f --- /dev/null +++ b/docs/dev/unify-logging.md @@ -0,0 +1,70 @@ +# Unified Logging + +This document describes the logging improvements in issue: + +## Problem Statement + +**Previously:** + +- Logging logic was duplicated across files +- Log formatting made logs unreadable +- Different colours for the same log level across jobs +- No log level control on Instance Project + +**Now:** + +- Logging is centralized +- Output is more readable and consistent +- Colour is consistent across jobs +- Input pipeline parameters logged on each job +- Log level can be set on Instance Project + +--- + +## Key Changes + +### 1. Centralized Logging + +A single logging module was introduced. +All other modules import and use it. + +**Benefits:** + +- No duplicated logging logic +- Easier maintenance +- Consistent format across the repository +- Usage of one logging library (`import logging`) + +[Link to file](https://github.com/Netcracker/qubership-envgene/blob/main/python/envgene/envgenehelper/logger.py) + +--- + +### 2. New Logging Parameter + +A new parameter was added to control logging behavior. + +[Link to documentation](https://github.com/Netcracker/qubership-envgene/blob/a823f450a671d058813991b218b9afde59f6db41/docs/envgene-repository-variables.md#envgene_log_level) + +--- + +### 3. Coloured Log Levels + +Different log levels now have consistent colours across jobs: + +| Level | Colour | +|----------------|--------| +| DEBUG | Blue | +| INFO (default) | White | +| WARNING | Yellow | +| ERROR | Red | + +--- + +### 4. Parameter Logging Script for Generated Jobs + +A script was added that: + +- Runs at the start of every generated job +- Logs input parameters + +[How it was implemented](https://github.com/Netcracker/qubership-envgene/blob/main/build_pipegene/scripts/pipeline_helper.py#L47-L50) diff --git a/docs/envgene-objects.md b/docs/envgene-objects.md index d598cf25e..4648931cd 100644 --- a/docs/envgene-objects.md +++ b/docs/envgene-objects.md @@ -31,6 +31,7 @@ - [Environment Credentials File](#environment-credentials-file) - [Shared Credentials File](#shared-credentials-file) - [System Credentials File (in Instance repository)](#system-credentials-file-in-instance-repository) + - [Shared Template Variable Files](#shared-template-variable-files) - [Environment Specific ParameterSet](#environment-specific-parameterset) - [Environment Specific Resource Profile Override](#environment-specific-resource-profile-override) - [Cloud Passport](#cloud-passport) @@ -1134,6 +1135,40 @@ gitlab-token-cred: secret: "token-placeholder-123" ``` +### Shared Template Variable Files + +This file includes template variables that can be shared across multiple environments. During Environment Instance generation, EnvGene automatically merges variables from these shared files with `additionalTemplateVariables` from the [Environment Inventory](/docs/envgene-configs.md#env_definitionyml). + +The relationship between Shared Template Variable Files and Environment is established through: + +- The `envTemplate.sharedTemplateVariables` property in [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) +- The property value should be the filename (without extension) of the Shared Template Variable File + +Files can be defined at three scopes with different precedence: + +1. **Environment-level** + **Location:** `/environments///shared-template-variables/` +2. **Cluster-level** + **Location:** `/environments//shared-template-variables/` +3. **Site-level** + **Location:** `/environments/shared-template-variables/` + +EnvGene checks these locations in order (environment → cluster → site) and uses the first matching file found. + +The file must contain a key-value hashmap and must NOT be located in a `parameters` directory. + +During Environment Instance generation, variables from Shared Template Variable Files are merged with `additionalTemplateVariables` from the Environment Inventory. Variables from `additionalTemplateVariables` take precedence over variables from Shared Template Variable Files if there are conflicts. + +**Example:** + +```yaml +TEMPLATE_VARIABLE_1: "value-1" +TEMPLATE_VARIABLE_2: "value-2" +nested: + key1: "nested-value-1" + key2: "nested-value-2" +``` + ### Environment Specific ParameterSet TBD diff --git a/docs/envgene-pipelines.md b/docs/envgene-pipelines.md index 1a3d47a8c..35971e4d4 100644 --- a/docs/envgene-pipelines.md +++ b/docs/envgene-pipelines.md @@ -22,6 +22,11 @@ If multiple [`ENV_NAMES`](/docs/instance-pipeline-parameters.md#env_names) are s ### [Instance pipeline] Job sequence +```mermaid +flowchart LR + A[trigger_passport] --> B[get_passport] --> C[process_decryption_mode] --> D[env_inventory_generation] --> E[credential_rotation] --> F[app_reg_def_process] --> G[process_sd] --> H[env_build] --> I[generate_effective_set] --> J[git_commit] +``` + 1. **bg_manage** - **Condition**: Runs if [`BG_MANAGE: true`](/docs/instance-pipeline-parameters.md#bg_manage). - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) @@ -35,25 +40,45 @@ If multiple [`ENV_NAMES`](/docs/instance-pipeline-parameters.md#env_names) are s - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) 4. **env_inventory_generation**: - - **Condition**: Runs if [`ENV_TEMPLATE_TEST: false`](/docs/instance-pipeline-parameters.md#env_template_test) AND ([`ENV_SPECIFIC_PARAMETERS`](/docs/instance-pipeline-parameters.md#env_specific_params) OR [`ENV_TEMPLATE_NAME`](/docs/instance-pipeline-parameters.md#env_template_name)) + - **Condition**: Runs if [`ENV_TEMPLATE_TEST: false`](/docs/instance-pipeline-parameters.md#env_template_test) AND ([`ENV_SPECIFIC_PARAMS`](/docs/instance-pipeline-parameters.md#env_specific_params) OR [`ENV_TEMPLATE_NAME`](/docs/instance-pipeline-parameters.md#env_template_name)) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) 5. **credential_rotation**: - **Condition**: Runs if [`CRED_ROTATION_PAYLOAD`](/docs/instance-pipeline-parameters.md#cred_rotation_payload) is provided - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) -6. **process_sd**: +6. **app_reg_def_process**: + - **What happens in this job**: + 1. Handles certificate updates from the configuration directory. + 2. Downloads the Environment Template artifact. + 3. Renders [Application Definitions](/docs/envgene-objects.md#application-definition) and [Registry Definitions](/docs/envgene-objects.md#registry-definition) from: + 1. Templates, as described in [User Defined by Template](/docs/features/app-reg-defs.md#user-defined-by-template) + 2. External Job, as described in [External Job](/docs/features/app-reg-defs.md#external-job) + 4. Runs [Application and Registry Definitions Transformation](/docs/features/app-reg-defs.md#application-and-registry-definitions-transformation) + - **Condition**: Runs if ( [`ENV_BUILD: true`](/docs/instance-pipeline-parameters.md#env_builder) ) + - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) + +7. **process_sd**: - **Condition**: Runs if ( [`SOURCE_TYPE: json`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_DATA`](/docs/instance-pipeline-parameters.md#sd_data) is provided ) OR ( [`SOURCE_TYPE: artifact`](/docs/instance-pipeline-parameters.md#sd_source_type) AND [`SD_VERSIONS`](/docs/instance-pipeline-parameters.md#sd_version) is provided ) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) -7. **env_build**: +8. **env_build**: + - **What happens in this job**: + 1. Handles certificate updates from the configuration directory. + 2. Downloads the Environment Template artifact from the `app_reg_def_process` job artifacts, **not from the registry** + 3. Updates the Environment Template version if [`ENV_TEMPLATE_VERSION`](/docs/instance-pipeline-parameters.md#env_template_version) is provided. + 4. Renders the environment using Jinja2 templates (renders Namespaces, Clouds, and other environment components, but not Application and Registry Definitions). + 5. Handles template overrides + 6. Handles template Parameter Set and Resource profiles. + 7. Handles environment-specific Parameter Set and Resource profiles. + 8. Creates Credentials including shared Credentials - **Condition**: Runs if [`ENV_BUILD: true`](/docs/instance-pipeline-parameters.md#env_builder). - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) -8. **generate_effective_set**: +9. **generate_effective_set**: - **Condition**: Runs if [`GENERATE_EFFECTIVE_SET: true`](/docs/instance-pipeline-parameters.md#generate_effective_set) - **Docker image**: [`qubership-effective-set-generator`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-effective-set-generator) -9. **git_commit**: +10. **git_commit**: - **Condition**: Runs if there are jobs requiring changes to the repository AND [`ENV_TEMPLATE_TEST: false`](/docs/instance-pipeline-parameters.md#env_template_test) - **Docker image**: [`qubership-envgene`](https://github.com/Netcracker/qubership-envgene/pkgs/container/qubership-envgene) diff --git a/docs/features/app-reg-defs.md b/docs/features/app-reg-defs.md index 7b952a725..52f141ad4 100644 --- a/docs/features/app-reg-defs.md +++ b/docs/features/app-reg-defs.md @@ -48,7 +48,7 @@ There are two sources for obtaining Application and Registry Definitions in EnvG An external job (not implemented in EnvGene itself, but serves as an extension point) that somehow creates/discovers/generates Application and Registry Definitions as YAML files and saves them in its artifact with the contract name `definitions.zip`. -During the generation of an Environment Instance (as part of `env_build_job`), EnvGene retrieves the Application and Registry Definitions from this artifact and saves them as part of the environment instance. +During the [`app_reg_def_process`](/docs/envgene-pipelines.md#instance-pipeline) job execution, EnvGene retrieves the Application and Registry Definitions from this artifact and saves them as part of the Environment instance. EnvGene uses the following instance repository pipeline parameters: @@ -79,7 +79,7 @@ These files can be either Jinja templates of the object or plain objects without Each Application and Registry Definition is created as a separate file. -During Environment Instance generation (as part of `env_build_job`), EnvGene renders these templates and saves them as part of the Environment Instance. +During the [`app_reg_def_process`](/docs/envgene-pipelines.md#instance-pipeline) job execution, EnvGene renders these templates and saves them as part of the Environment Instance. ### Using Application and Registry Definitions diff --git a/docs/features/calculator-cli.md b/docs/features/calculator-cli.md index fae8d720e..81a72ee29 100644 --- a/docs/features/calculator-cli.md +++ b/docs/features/calculator-cli.md @@ -1,4 +1,3 @@ - # Calculator CLI - [Calculator CLI](#calculator-cli) @@ -25,6 +24,10 @@ - [\[Version 2.0\] App chart validation](#version-20-app-chart-validation) - [\[Version 2.0\] Sensitive parameters processing](#version-20-sensitive-parameters-processing) - [\[Version 2.0\] No SBOMs Mode](#version-20-no-sboms-mode) + - [\[Version 2.0\] Parameters Priority](#version-20-parameters-priority) + - [\[Version 2.0\] Traceability Comments](#version-20-traceability-comments) + - [Parameter Source to Comment Mapping](#parameter-source-to-comment-mapping) + - [Rules for Adding Comments](#rules-for-adding-comments) - [\[Version 2.0\] Deployment Parameter Context](#version-20-deployment-parameter-context) - [\[Version 2.0\]\[Deployment Parameter Context\] `deployment-parameters.yaml`](#version-20deployment-parameter-context-deployment-parametersyaml) - [\[Version 2.0\] Predefined `deployment-parameters.yaml` parameters](#version-20-predefined-deployment-parametersyaml-parameters) @@ -95,18 +98,19 @@ Below is a **complete** list of attributes -| Attribute | Type | Mandatory | Description | Default | Example | -|---|---|---|---|---|--| -| `--env-id`/`-e` | string | yes | Environment ID in `/` notation | N/A | `cluster/platform-00` | -| `--envs-path`/`-ep` | string | yes | Path to `/environments` folder | N/A | `/environments` | -| `--sboms-path`/`-sp`| string | no | Path to the folder with Application SBOMs. If the attribute is not provided, generation occurs in [No SBOMs Mode](#version-20-no-sboms-mode) | N/A |`/sboms` | -| `--sd-path`/`-sdp`| string | yes | Path to the Solution Descriptor | N/A | `/environments/cluster/platform-00/Inventory/solution-descriptor/sd.yaml` | -| `--registries`/`-r`| string | no | Required when `--sd-path` and `--sboms-path` are provided. Optional for [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/configuration/registry.yml` | -| `--output`/`-o` | string | yes | Folder where the result will be put by Calculator command-line tool | N/A | `/environments/cluster/platform-00/effective-set` | -| `--effective-set-version`/`-esv` | string | no | The version of the effective set to be generated. Available options are `v1.0` and `v2.0` | `v2.0` | `v1.0` | -| `--pipeline-consumer-specific-schema-path`/`-pcssp` | string | no | Path to a JSON schema defining a consumer-specific pipeline context component. Multiple attributes of this type can be provided | N/A | | -| `--extra_params`/`-ex` | string | no | Additional parameters used by the Calculator for effective set generation. Multiple instances of this attribute can be provided | N/A | `DEPLOYMENT_SESSION_ID=550e8400-e29b-41d4-a716-446655440000` | -| `--app_chart_validation`/`-acv` | boolean | no | Determines whether [app chart validation](#version-20-app-chart-validation) should be performed. If `true` validation is enabled (checks for `application/vnd.qubership.app.chart` in SBOM). If `false` validation is skipped | `true` | `false` | +| Attribute | Type | Mandatory | Description | Default | Example | +|-----------------------------------------------------|---------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------| +| `--env-id`/`-e` | string | yes | Environment ID in `/` notation | N/A | `cluster/platform-00` | +| `--envs-path`/`-ep` | string | yes | Path to `/environments` folder | N/A | `/environments` | +| `--sboms-path`/`-sp` | string | no | Path to the folder with Application SBOMs. If the attribute is not provided, generation occurs in [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/sboms` | +| `--sd-path`/`-sdp` | string | yes | Path to the Solution Descriptor | N/A | `/environments/cluster/platform-00/Inventory/solution-descriptor/sd.yaml` | +| `--registries`/`-r` | string | no | Required when `--sd-path` and `--sboms-path` are provided. Optional for [No SBOMs Mode](#version-20-no-sboms-mode) | N/A | `/configuration/registry.yml` | +| `--output`/`-o` | string | yes | Folder where the result will be put by Calculator command-line tool | N/A | `/environments/cluster/platform-00/effective-set` | +| `--effective-set-version`/`-esv` | string | no | The version of the effective set to be generated. Available options are `v1.0` and `v2.0` | `v2.0` | `v1.0` | +| `--pipeline-consumer-specific-schema-path`/`-pcssp` | string | no | Path to a JSON schema defining a consumer-specific pipeline context component. Multiple attributes of this type can be provided | N/A | | +| `--extra_params`/`-ex` | string | no | Additional parameters used by the Calculator for effective set generation. Multiple instances of this attribute can be provided | N/A | `DEPLOYMENT_SESSION_ID=550e8400-e29b-41d4-a716-446655440000` | +| `--app_chart_validation`/`-acv` | boolean | no | Determines whether [app chart validation](#version-20-app-chart-validation) should be performed. If `true` validation is enabled (checks for `application/vnd.qubership.app.chart` in SBOM). If `false` validation is skipped | `true` | `false` | +| `--enable-traceability`/`-etr` | boolean | no | Determines whether [traceability](#version-20-traceability-comments) will be enabled. If `true`, traceability comments will be added. If `false`, they will be omitted. | `false` | `true` | ### Registry Configuration @@ -479,6 +483,123 @@ The Calculator generates only the following contexts: These contexts are used as a source exclusively from the environment instance. +#### [Version 2.0] Parameters Priority + +Parameters in the Effective Set come from different sources: + +1. User-defined in the [Environment Instance](/docs/envgene-objects.md#environment-instance-objects) +2. [Application SBOM](/docs/sbom.md#application-sbom) +3. Generated by the calculator +4. Calculator extra parameters (`--extra_params`) +5. Calculator defaults + +Parameters can be set at different levels: + +1. Service level +2. Application level +3. Namespace level +4. Cloud level +5. Tenant level + +If the same parameter key is set in multiple sources or levels, the Calculator uses the following priority (from highest to lowest): + +1. User-defined in [Resource Profile Override](/docs/envgene-objects.md#resource-profile-override-in-instance) of the Environment Instance +2. Service level defined in Resource Profile Baseline in Application SBOM +3. Service level defined in other Application SBOM attributes +4. Calculator-generated at the Service level +5. User-defined at the [Application](/docs/envgene-objects.md#application) level in Environment Instance +6. Application level defined in Application SBOM +7. Calculator-generated at the Application level +8. User-defined at the [Namespace](/docs/envgene-objects.md#namespace) level in Environment Instance +9. Namespace level defined in Application SBOM +10. Calculator-generated at the Namespace level +11. User-defined at the [Cloud](/docs/envgene-objects.md#cloud) level in Environment Instance +12. Cloud level defined in Application SBOM +13. Calculator-generated at the Cloud level +14. User-defined at the [Tenant](/docs/envgene-objects.md#tenant) level in Environment Instance +15. Calculator extra parameters (`--extra_params`) +16. Default values by calculator + +For example, if a user sets `BASELINE_PROJ` at the Namespace level, this value will override the value calculated by the calculator. If `BASELINE_PROJ` is set at the cloud level, it will be overridden by calculator-generated values at Namespace or Application levels. + +Not every parameter in the Effective Set can be overridden. Overriding service-level parameters is only possible via [resource profile override](/docs/envgene-objects.md#resource-profile-override-in-instance). + +> [!NOTE] +> If a complex parameter is set, the value from the higher-priority source **completely replaces** the lower-priority one; merging is not performed. + +#### [Version 2.0] Traceability Comments + +To make it clear where each parameter in the Effective Set comes from, comments are added next to every value showing its source. + +The CLI flag [`--enable-traceability`](#calculator-command-line-tool-execution-attributes) enables these comments in Effective Set. + +##### Parameter Source to Comment Mapping + +| Parameter Source | Comment | Example | +|----------------------------------------------------|--------------------------------------------|----------------------------------------------------------------------------------------------| +| Environment Instance, Tenant | `# tenant` | `GITLAB_URL: "https://git.qibership.org" # tenant` | +| Environment Instance, Cloud | `# cloud` | `CLOUD_API_HOST: "https://api.example.com" # cloud` | +| Environment Instance, Namespace | `# namespace: ` | `NAMESPACE_NAME: "env-1-core" # namespace: env-1-core` | +| Environment Instance, Application | `# application: ` | `APP_FEATURE_FLAG: true # application: my-app` | +| Environment Instance, Resource Profile Override | `# resource-profile-override: ` | `CPU_LIMIT: "500m" # resource-profile-override: perf-small` | +| Environment Instance, Composite Structure | `# composite-structure` | `composite_structure: # composite-structure` | +| Environment Instance, BG Domain | `# bg-domain` | `bg_domain: # bg-domain` | +| Application SBOM | `# sbom` | `deploy_param: '' # sbom` | +| Application SBOM, Resource Profile Baseline | `# sbom, resource-profile-baseline: `| `PROFILE_BASELINE: "dev" # sbom, resource-profile-baseline: dev` | +| Calculated by calculator | `# envgene calculated` | `PUBLIC_GATEWAY_URL: "https://public-gateway-bss.qubership.org" # envgene calculated` | +| Calculator `--extra_params` | `# envgene pipeline parameter` | `DEPLOYMENT_SESSION_ID: "7e9f5f54-4be2-4fbd-a267-19e78d09810d" # envgene pipeline parameter` | +| Default value by calculator | `# envgene default` | `MANAGED_BY: "argocd" # envgene default` | + +##### Rules for Adding Comments + +1. The comment is added after a single space following the parameter value on the same line for non-multiline values. + + ```yaml + SECURITY_POLICY: strict # cloud + ``` + +2. The comment is added on the previous line above the parameter for multiline values (using `|` or `>`). + + ```yaml + # cloud + CS_CONTENT_SECURITY_POLICY: | + {"CONTENT_SECURITY_POLICY":"default-src..."} + ``` + +3. Comments are added in all Effective Set contexts. + +4. Comments are not added in `mapping.yaml` files. + +5. The source is determined by [priority](#version-20-parameters-priority): if a parameter is defined in multiple sources, the source with the highest priority is used. + +6. A parameter is considered calculated by the calculator if its value has been changed by the calculator (transformed, generated, obtained via concatenation, etc.) + +7. For complex parameters calculated by the calculator, the comment is added only to the root parameter. + +8. Complex parameters (objects, lists) are processed recursively, with each item receiving a comment based on its own source. + + ```yaml + servers: + - "server1.example.com" # cloud + - "server2.example.com" # namespace: env-1 + + servers: + name: "server1" # cloud + host: "host1" # cloud + port: 8080 # namespace: env-1 + ``` + +9. Comments are not added to YAML anchors/aliases (`&id001`, `*id001`, `<<: *id001`). But for regular keys/values filled in via anchors/aliases, still show their source as a comment: + + ```yaml + global: &id001 + key1: value1 # cloud + key2: value2 # cloud + service1: + <<: *id001 + key3: value3 # namespace: env-1-core + ``` + #### [Version 2.0] Deployment Parameter Context These parameters establish a dedicated rendering context exclusively applied during application (re)deployment operations for Helm manifest rendering. @@ -555,7 +676,7 @@ The `` can be complex, such as a map or a list, whose elements can also b | `GATEWAY_URL` | yes | string | **Deprecated**. Internal Gateway URL | `http://internal-gateway-service:8080` | N/A | | `STATIC_CACHE_SERVICE_ROUTE_HOST` | yes | string | **Deprecated**. Constructed as static-cache-service-`NAMESPACE`.`CLOUD_PUBLIC_HOST` | None | N/A | | `BUILD_TAG_NEW` | yes | string | TBD | `keycloak-database` | N/A | -| `SSL_SECRET` | yes | string | Specifies the name of the Kubernetes secret that holds the SSL certificate bundle. This parameter can be explicitly set by the user at the `Tenant`, `Cloud`, `Namespace`, or `Application` level. If not provided, the default value will be used. | `defaultsslcertificate` | N/A | +| `SSL_SECRET` | yes | string | Specifies the name of the Kubernetes secret that holds the SSL certificate bundle. This parameter can be explicitly set by the user at the `Tenant`, `Cloud`, `Namespace`, or `Application` level. If not provided, the default value will be used. | `defaultsslcertificate` | N/A | | `CERTIFICATE_BUNDLE_MD5SUM` | no | string | Hash sum of the value provided in `DEFAULT_SSL_CERTIFICATES_BUNDLE`, calculated using the MD5 algorithm | None | N/A | | `ORIGIN_NAMESPACE` | yes | string | The name of the origin namespace for the [BG Domain](/docs/envgene-objects.md#bg-domain). If no BG Domain is present, this defaults to the value of `${NAMESPACE}` | `${NAMESPACE}` | N/A | | `PEER_NAMESPACE` | yes | string | The name of the peer namespace for the [BG Domain](/docs/envgene-objects.md#bg-domain). If no BG Domain is present, this defaults to the value of `${NAMESPACE}` | `${NAMESPACE}` | N/A | @@ -566,8 +687,8 @@ The `` can be complex, such as a map or a list, whose elements can also b | `BASELINE_PROJ` | no | string | **Deprecated** | TBD | N/A | | `PUBLIC_GATEWAY_URL` | yes | string | URL of the public gateway for the namespace. The value is calculated using `${ORIGIN_NAMESPACE}` to have the same value for Origin, Peer and Controller namespaces in case when namespace belongs to [BG Domain](/docs/envgene-objects.md#bg-domain). Computed as `${CLOUD_PROTOCOL}://${PUBLIC_GATEWAY_ROUTE_HOST}` if `${PUBLIC_GATEWAY_ROUTE_HOST}` is set; otherwise, `${CLOUD_PROTOCOL}://public-gateway-${ORIGIN_NAMESPACE}.${CLOUD_PUBLIC_HOST}` | `${CLOUD_PROTOCOL}://public-gateway-${ORIGIN_NAMESPACE}.${CLOUD_PUBLIC_HOST}` | N/A | | `PRIVATE_GATEWAY_URL` | yes | string | URL of the private gateway for the namespace. The value is calculated using `${ORIGIN_NAMESPACE}` to have the same value for Origin, Peer and Controller namespaces in case when namespace belongs to [BG Domain](/docs/envgene-objects.md#bg-domain). Computed as `${CLOUD_PROTOCOL}://${PRIVATE_GATEWAY_ROUTE_HOST}` if `${PRIVATE_GATEWAY_ROUTE_HOST}` is set; otherwise, `${CLOUD_PROTOCOL}://private-gateway-${ORIGIN_NAMESPACE}.${CLOUD_PUBLIC_HOST}` | `${CLOUD_PROTOCOL}://private-gateway-${ORIGIN_NAMESPACE}.${CLOUD_PUBLIC_HOST}` | N/A | -| `PUBLIC_IDENTITY_PROVIDER_URL` | yes | string | URL of the public gateway for the IDP namespace. For namespaces in a [Composite Structure](/docs/envgene-objects.md#composite-structure), this points to the baseline namespace’s public gateway even if the current namespace is a satellite. Computed as URL of the public gateway where the IDP is published: Use the value of `PUBLIC_IDENTITY_PROVIDER_URL` from the `BASELINE_ORIGIN` namespace if `BASELINE_ORIGIN` is defined; otherwise, use `${PUBLIC_GATEWAY_URL}` from the current namespace | `${PUBLIC_GATEWAY_URL}` | N/A | -| `PRIVATE_IDENTITY_PROVIDER_URL` | yes | string | URL of the private gateway for the IDP namespace. For namespaces in a [Composite Structure](/docs/envgene-objects.md#composite-structure), this points to the baseline namespace’s private gateway even if the current namespace is a satellite. Computed as the URL of the private gateway where the IDP is published: Use the value of `PRIVATE_IDENTITY_PROVIDER_URL` from the `BASELINE_ORIGIN` namespace if `BASELINE_ORIGIN` is defined; otherwise, use `${PRIVATE_GATEWAY_URL}` from the current namespace | `${PRIVATE_GATEWAY_URL}` | N/A | +| `PUBLIC_IDENTITY_PROVIDER_URL` | yes | string | URL of the public gateway for the IDP namespace. For namespaces in a [Composite Structure](/docs/envgene-objects.md#composite-structure), this points to the baseline namespace's public gateway even if the current namespace is a satellite. Computed as URL of the public gateway where the IDP is published: Use the value of `PUBLIC_IDENTITY_PROVIDER_URL` from the `BASELINE_ORIGIN` namespace if `BASELINE_ORIGIN` is defined; otherwise, use `${PUBLIC_GATEWAY_URL}` from the current namespace | `${PUBLIC_GATEWAY_URL}` | N/A | +| `PRIVATE_IDENTITY_PROVIDER_URL` | yes | string | URL of the private gateway for the IDP namespace. For namespaces in a [Composite Structure](/docs/envgene-objects.md#composite-structure), this points to the baseline namespace's private gateway even if the current namespace is a satellite. Computed as the URL of the private gateway where the IDP is published: Use the value of `PRIVATE_IDENTITY_PROVIDER_URL` from the `BASELINE_ORIGIN` namespace if `BASELINE_ORIGIN` is defined; otherwise, use `${PRIVATE_GATEWAY_URL}` from the current namespace | `${PRIVATE_GATEWAY_URL}` | N/A | > [!IMPORTANT] > Parameters whose keys match the name of one of the services must be excluded from this file @@ -702,66 +823,66 @@ Below are the descriptions of predefined parameters. Common Predefined Parameters: -| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | -|---|---|---|---|---|---| -| `APPLICATION_NAME` | yes | string | Name of the application | None | `.metadata.component.name` | -| `DEPLOYMENT_SESSION_ID` | yes | string | Effective Set calculation operation ID | None | Taken from input parameter `DEPLOYMENT_SESSION_ID` passed via `extra_params` (not from SBOM) | -| `MANAGED_BY` | yes | string | Deployer type. Always `argocd` | `argocd` | None | +| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | +|------------------------|-----------|--------|-----------------------------------------|-----------|----------------------------------------------------------------------------------------------| +| `APPLICATION_NAME` | yes | string | Name of the application | None | `.metadata.component.name` | +| `DEPLOYMENT_SESSION_ID`| yes | string | Effective Set calculation operation ID | None | Taken from input parameter `DEPLOYMENT_SESSION_ID` passed via `extra_params` (not from SBOM) | +| `MANAGED_BY` | yes | string | Deployer type. Always `argocd` | `argocd` | None | **Image Type** Service Predefined Parameters: -| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | -|---|---|---|---|---|---| -| `artifacts` | yes | list | always `[]` | `[]` | None | -| `deploy_param` | yes | string | None | `""` | `.components[?name=].properties[?name=deploy_param].value` | -| `docker_digest` | yes | string | Docker image checksum for the service, calculated using `SHA-256` algorithm | `""` | `.components[?name=].components[?mime-type=application/vnd.docker.image].hashes[0].content` | -| `docker_registry` | yes | string | None | None | `.components[?name=].properties[?name=docker_registry].value` | -| `docker_repository_name` | yes | string | The registry repository where the Docker image is located | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].group` | -| `docker_tag` | yes | string | Docker image version | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].version` | -| `full_image_name` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value` | -| `git_branch` | yes | string | Source code branch name used for service build | None | `.components[?name=].properties[?name=git_branch].value` | -| `git_revision` | yes | string | Git revision of the repository used for the build | None | `.components[?name=].properties[?name=git_revision].value` | -| `git_url` | yes | string | None | None | `.components[?name=].properties[?name=git_url].value` | -| `image` | yes | string | The same as `full_image_name` | None | `.components[?name=].properties[?name=full_image_name].value` | -| `image_name` | yes | string | Docker image name | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].name` | -| `image_type` | yes | string | None | None | `.components[?name=].properties[?name=image_type].value` | -| `name` | yes | string | Service name | None | `` | -| `promote_artifacts` | yes | bool | None | None | `.components[?name=].properties[?name=promote_artifacts].value` | -| `qualifier` | yes | string | None | None | `.components[?name=].properties[?name=qualifier].value` | -| `version` | yes | string | Service version | None | `.components[?name=].version` | +| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | +|-------------------------|-----------|--------|------------------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------| +| `artifacts` | yes | list | always `[]` | `[]` | None | +| `deploy_param` | yes | string | None | `""` | `.components[?name=].properties[?name=deploy_param].value` | +| `docker_digest` | yes | string | Docker image checksum for the service, calculated using `SHA-256` algorithm | `""` | `.components[?name=].components[?mime-type=application/vnd.docker.image].hashes[0].content` | +| `docker_registry` | yes | string | None | None | `.components[?name=].properties[?name=docker_registry].value` | +| `docker_repository_name`| yes | string | The registry repository where the Docker image is located | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].group` | +| `docker_tag` | yes | string | Docker image version | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].version` | +| `full_image_name` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value` | +| `git_branch` | yes | string | Source code branch name used for service build | None | `.components[?name=].properties[?name=git_branch].value` | +| `git_revision` | yes | string | Git revision of the repository used for the build | None | `.components[?name=].properties[?name=git_revision].value` | +| `git_url` | yes | string | None | None | `.components[?name=].properties[?name=git_url].value` | +| `image` | yes | string | The same as `full_image_name` | None | `.components[?name=].properties[?name=full_image_name].value` | +| `image_name` | yes | string | Docker image name | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].name` | +| `image_type` | yes | string | None | None | `.components[?name=].properties[?name=image_type].value` | +| `name` | yes | string | Service name | None | `` | +| `promote_artifacts` | yes | bool | None | None | `.components[?name=].properties[?name=promote_artifacts].value` | +| `qualifier` | yes | string | None | None | `.components[?name=].properties[?name=qualifier].value` | +| `version` | yes | string | Service version | None | `.components[?name=].version` | **Config Type** Service Predefined Parameters: -| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | -|------------------------------|-----------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| `artifact` | no | string | [Primary Service Artifact](#version-20-primary-service-artifact) | None | None | -| `artifact.artifactId` | no | string | artifact ID of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.name` | -| `artifact.groupId` | no | string | group ID of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.group` | -| `artifact.version` | no | string | version of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.version` | -| `artifacts` | yes | list | This section defines microservice artifacts. Artifacts are only populated for services/SBOM components that meet [specified conditions](#version-20-service-artifacts). All other cases should return `[]` | `[]` | None | -| `artifacts[].artifact_id` | yes | string | always `''` | `''` | None | -| `artifacts[].artifact_path` | yes | string | always `''` | `''` | None | -| `artifacts[].artifact_type` | yes | string | always `''` | `''` | None | -| `artifacts[].classifier` | yes | string | None | None | `.components[?name=].components[].properties[?name=classifier].value` | -| `artifacts[].deploy_params` | yes | string | always `''` | `''` | None | -| `artifacts[].gav` | yes | string | always `''` | `''` | None | -| `artifacts[].group_id` | yes | string | always `''` | `''` | None | -| `artifacts[].id` | yes | string | GAV coordinates of the artifact. Constructed by concatenating the `group`, `name`, and `version` attributes using `:` as separator | None | `.components[?name=].components[].group`:`.components[?name=].components[].name`:`.components[?name=].components[].version` | -| `artifacts[].name` | yes | string | Constructed by concatenating the `name`, `version` and `type` attributes using `-` and `.` as separator | None | `.components[?name=].components[].name`-`.components[?name=].components[].version`.`.components[?name=].components[].properties[?name=type].value` | -| `artifacts[].repository` | yes | string | always `''` | `''` | None | -| `artifacts[].type` | yes | string | None | None | `.components[?name=].components[].properties[?name=type].value` | -| `artifacts[].url` | yes | string | always `''` | `''` | None | -| `artifacts[].version` | yes | string | always `''` | `''` | None | -| `build_id_dtrust` | yes | string | None | None | `.components[?name=].components[].properties[?name=build_id_dtrust].value` | -| `git_branch` | yes | string | Source code branch name used for service build | None | `.components[?name=].properties[?name=git_branch].value` | -| `git_revision` | yes | string | Git revision of the repository used for the build | None | `.components[?name=].properties[?name=git_revision].value` | -| `git_url` | yes | string | None | None | `.components[?name=].components[].properties[?name=git_url].value` | -| `maven_repository` | yes | string | None | None | `.components[?name=].components[].properties[?name=maven_repository].value` | -| `name` | yes | string | Service name | None | `` | -| `service_name` | yes | string | Service name | None | `` | -| `tArtifactNames` | yes | hashmap | This section defines microservice ZIP artifacts. Artifacts are only populated for services/SBOM components that meet [specified conditions](#version-20-service-artifacts). All other cases should return `{}` Described in [`tArtifactNames`](#version-20-tartifactnames) | `{}` | None | -| `type` | no | string | None | None | `.components[?name=].components[].properties[?name=type].value` | -| `version` | yes | string | Service version | None | `.components[?name=].version` | +| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | +|-----------------------------|-----------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `artifact` | no | string | [Primary Service Artifact](#version-20-primary-service-artifact) | None | None | +| `artifact.artifactId` | no | string | artifact ID of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.name` | +| `artifact.groupId` | no | string | group ID of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.group` | +| `artifact.version` | no | string | version of [Primary Service Artifact](#version-20-primary-service-artifact) | None | `.version` | +| `artifacts` | yes | list | This section defines microservice artifacts. Artifacts are only populated for services/SBOM components that meet [specified conditions](#version-20-service-artifacts). All other cases should return `[]` | `[]` | None | +| `artifacts[].artifact_id` | yes | string | always `''` | `''` | None | +| `artifacts[].artifact_path` | yes | string | always `''` | `''` | None | +| `artifacts[].artifact_type` | yes | string | always `''` | `''` | None | +| `artifacts[].classifier` | yes | string | None | None | `.components[?name=].components[].properties[?name=classifier].value` | +| `artifacts[].deploy_params` | yes | string | always `''` | `''` | None | +| `artifacts[].gav` | yes | string | always `''` | `''` | None | +| `artifacts[].group_id` | yes | string | always `''` | `''` | None | +| `artifacts[].id` | yes | string | GAV coordinates of the artifact. Constructed by concatenating the `group`, `name`, and `version` attributes using `:` as separator | None | `.components[?name=].components[].group`:`.components[?name=].components[].name`:`.components[?name=].components[].version` | +| `artifacts[].name` | yes | string | Constructed by concatenating the `name`, `version` and `type` attributes using `-` and `.` as separator | None | `.components[?name=].components[].name`-`.components[?name=].components[].version`.`.components[?name=].components[].properties[?name=type].value` | +| `artifacts[].repository` | yes | string | always `''` | `''` | None | +| `artifacts[].type` | yes | string | None | None | `.components[?name=].components[].properties[?name=type].value` | +| `artifacts[].url` | yes | string | always `''` | `''` | None | +| `artifacts[].version` | yes | string | always `''` | `''` | None | +| `build_id_dtrust` | yes | string | None | None | `.components[?name=].components[].properties[?name=build_id_dtrust].value` | +| `git_branch` | yes | string | Source code branch name used for service build | None | `.components[?name=].properties[?name=git_branch].value` | +| `git_revision` | yes | string | Git revision of the repository used for the build | None | `.components[?name=].properties[?name=git_revision].value` | +| `git_url` | yes | string | None | None | `.components[?name=].components[].properties[?name=git_url].value` | +| `maven_repository` | yes | string | None | None | `.components[?name=].components[].properties[?name=maven_repository].value` | +| `name` | yes | string | Service name | None | `` | +| `service_name` | yes | string | Service name | None | `` | +| `tArtifactNames` | yes | hashmap | This section defines microservice ZIP artifacts. Artifacts are only populated for services/SBOM components that meet [specified conditions](#version-20-service-artifacts). All other cases should return `{}` Described in [`tArtifactNames`](#version-20-tartifactnames) | `{}` | None | +| `type` | no | string | None | None | `.components[?name=].components[].properties[?name=type].value` | +| `version` | yes | string | Service version | None | `.components[?name=].version` | ###### [Version 2.0] Service Artifacts @@ -905,28 +1026,28 @@ Set of per service keys depends on the service type, determined by the MIME type **Image Type** Per Service Parameters: -| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | -|---|---|---|---|---|---| -| `DEPLOYMENT_SESSION_ID` | yes | string | Effective Set calculation operation ID | None | Taken from input parameter `DEPLOYMENT_SESSION_ID` passed via `extra_params` (not from SBOM) | -| `MANAGED_BY` | yes | string | Deployer type. Always `argocd` | `argocd` | None | -| `ARTIFACT_DESCRIPTOR_VERSION` | yes | string | `.metadata.component.version` | None | None | -| `DEPLOYMENT_RESOURCE_NAME` | yes | string | Is formed by concatenating ``-v1 | None | None | -| `DEPLOYMENT_VERSION` | yes | string | always `v1` | `v1` | None | -| `DOCKER_TAG` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value` | -| `IMAGE_REPOSITORY` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value.split(':').join(parts[:2]` | -| `SERVICE_NAME` | yes | string | `` | None | None | -| `TAG` | yes | string | Docker image version | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].version` | +| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | +|------------------------------|-----------|--------|------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------| +| `DEPLOYMENT_SESSION_ID` | yes | string | Effective Set calculation operation ID | None | Taken from input parameter `DEPLOYMENT_SESSION_ID` passed via `extra_params` (not from SBOM) | +| `MANAGED_BY` | yes | string | Deployer type. Always `argocd` | `argocd`| None | +| `ARTIFACT_DESCRIPTOR_VERSION`| yes | string | `.metadata.component.version` | None | None | +| `DEPLOYMENT_RESOURCE_NAME` | yes | string | Is formed by concatenating ``-v1 | None | None | +| `DEPLOYMENT_VERSION` | yes | string | always `v1` | `v1` | None | +| `DOCKER_TAG` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value` | +| `IMAGE_REPOSITORY` | yes | string | None | None | `.components[?name=].properties[?name=full_image_name].value.split(':').join(parts[:2])` | +| `SERVICE_NAME` | yes | string | `` | None | None | +| `TAG` | yes | string | Docker image version | None | `.components[?name=].components[?mime-type=application/vnd.docker.image].version` | **Config Type** Per Service Parameters: -| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | -|---|---|---|---|---|---| -| `DEPLOYMENT_SESSION_ID` | yes | string | Effective Set calculation operation ID | None | Taken from input parameter `DEPLOYMENT_SESSION_ID` passed via `extra_params` (not from SBOM) | -| `MANAGED_BY` | yes | string | Deployer type. Always `argocd` | `argocd` | None | -| `ARTIFACT_DESCRIPTOR_VERSION` | yes | string | `.metadata.component.version` | None | None | -| `DEPLOYMENT_RESOURCE_NAME` | yes | string | Is formed by concatenating ``-v1 | None | None | -| `DEPLOYMENT_VERSION` | yes | string | always `v1` | `v1` | None | -| `SERVICE_NAME` | yes | string | `` | None | None | +| Attribute | Mandatory | Type | Description | Default | Source in Application SBOM | +|-------------------------------|-----------|--------|----------------------------------------------|---------|-------------------------------------------------------------------------------------------| +| `DEPLOYMENT_SESSION_ID` | yes | string | Effective Set calculation operation ID | None | Taken from the `DEPLOYMENT_SESSION_ID` input parameter via `extra_params` (not from SBOM) | +| `MANAGED_BY` | yes | string | Type of deployer. Always `argocd` | `argocd`| None | +| `ARTIFACT_DESCRIPTOR_VERSION` | yes | string | `.metadata.component.version` | None | None | +| `DEPLOYMENT_RESOURCE_NAME` | yes | string | Formed by concatenating ``-v1 | None | None | +| `DEPLOYMENT_VERSION` | yes | string | always `v1` | `v1` | None | +| `SERVICE_NAME` | yes | string | `` | None | None | ###### \[Version 2.0][Deployment Parameter Context] Resource Profile Processing @@ -1065,13 +1186,13 @@ For more information, refer to [Sensitive parameters processing](#version-20-sen This context only contains parameters generated by EnvGene: -| Attribute | Mandatory | Description | Default | Example | -|---|---|---|---|---| -| **composite_structure** | Mandatory | Contains the unmodified [Composite Structure](/docs/envgene-objects.md#composite-structure) object of the Environment Instance for which the Effective Set is generated. This variable is located in `parameters.yaml` | `{}`| [example](#version-20topology-context-composite_structure-example) | -| **k8s_tokens** | Mandatory | Contains deployment tokens for each namespace in the Environment Instance. The value is derived from the `data.secret` property of the Credential specified via `defaultCredentialsId` attribute in the corresponding `Namespace` or parent `Cloud`. If the attribute is not defined at the `Namespace` level, it is inherited from the parent `Cloud`. If defined at both levels, the `Namespace` value takes precedence. Either the `Cloud` or `Namespace` must define `defaultCredentialsId`. This variable is located in `credentials.yaml` | None | [example](#version-20topology-context-k8s_tokens-example) | -| **environments** | Mandatory | Contains **all** repository Environments, not just the one for which the Effective Set calculation was run. For each Environment, it includes the names of its contained namespaces. For each namespace, it provides a `deployPostfix` attribute. The `deployPostfix` value is derived from the namespace folder name (a child of `Namespaces` and parent of `namespace.yml`). For namespaces that are part of a BG Domain with roles `peer` or `origin`, the `deployPostfix` is obtained by removing the suffix `-peer` or `-origin` respectively from the namespace folder name. For all other namespaces (including `controller` namespace in BG Domain), the `deployPostfix` equals the namespace folder name. The namespace folder name is determined according to [Namespace Folder Name Generation](/docs/features/environment-instance-generation.md#namespace-folder-name-generation) rules. This variable is located in `parameters.yaml` | None | [example](#version-20topology-context-environments-example) | -| **cluster** | Mandatory | Contains information about the cluster where the Environment Instance is deployed. Includes cluster name, type, and other cluster-specific metadata taken from the [Cloud](/docs/envgene-objects.md#cloud) object. This variable is located in `parameters.yaml` | `{}` | [example](#version-20topology-context-cluster-example) | -| **bg_domain** | Mandatory | Contains the [BG Domain](/docs/envgene-objects.md#bg-domain) object from the Environment Instance for which the Effective Set is generated. Additionally, two extra sensitive attributes are added: `bg_domain.controllerNamespace.username` and `bg_domain.controllerNamespace.password`, whose values are taken from the [Credential](/docs/envgene-objects.md#credential) with `usernamePassword` type and the ID from the `bg_domain.controllerNamespace.credentials` attribute. The `credentials` attribute is removed. Non-sensitive parts of this variable are stored in `parameters.yaml`, while sensitive parts are stored in `credentials.yaml`. | `{}` | [example](#version-20topology-context-bg_domain-example) | +| Attribute | Mandatory | Description | Default | Example | +|-----------------------|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------| +| `composite_structure` | Mandatory | Contains the unmodified [Composite Structure](/docs/envgene-objects.md#composite-structure) object of the Environment Instance for which the Effective Set is generated. This variable is located in `parameters.yaml` | `{}` | [example](#version-20topology-context-composite_structure-example) | +| `k8s_tokens` | Mandatory | Contains deployment tokens for each namespace in the Environment Instance. The value is derived from the `data.secret` property of the Credential specified via `defaultCredentialsId` attribute in the corresponding `Namespace` or parent `Cloud`. If the attribute is not defined at the `Namespace` level, it is inherited from the parent `Cloud`. If defined at both levels, the `Namespace` value takes precedence. Either the `Cloud` or `Namespace` must define `defaultCredentialsId`. This variable is located in `credentials.yaml` | None | [example](#version-20topology-context-k8s_tokens-example) | +| `environments` | Mandatory | Contains **all** repository Environments, not just the one for which the Effective Set calculation was run. For each Environment, it includes the names of its contained namespaces. For each namespace, it provides a `deployPostfix` attribute. The `deployPostfix` value is derived from the namespace folder name (a child of `Namespaces` and parent of `namespace.yml`). For namespaces that are part of a BG Domain with roles `peer` or `origin`, the `deployPostfix` is obtained by removing the suffix `-peer` or `-origin` respectively from the namespace folder name. For all other namespaces (including `controller` namespace in BG Domain), the `deployPostfix` equals the namespace folder name. The namespace folder name is determined according to [Namespace Folder Name Generation](/docs/features/environment-instance-generation.md#namespace-folder-name-generation) rules. This variable is located in `parameters.yaml` | None | [example](#version-20topology-context-environments-example) | +| `cluster` | Mandatory | Contains information about the cluster where the Environment Instance is deployed. Includes cluster name, type, and other cluster-specific metadata taken from the [Cloud](/docs/envgene-objects.md#cloud) object. This variable is located in `parameters.yaml` | `{}` | [example](#version-20topology-context-cluster-example) | +| `bg_domain` | Mandatory | Contains the [BG Domain](/docs/envgene-objects.md#bg-domain) object from the Environment Instance for which the Effective Set is generated. Additionally, two extra sensitive attributes are added: `bg_domain.controllerNamespace.username` and `bg_domain.controllerNamespace.password`, whose values are taken from the [Credential](/docs/envgene-objects.md#credential) with `usernamePassword` type and the ID from the `bg_domain.controllerNamespace.credentials` attribute. The `credentials` attribute is removed. Non-sensitive parts of this variable are stored in `parameters.yaml`, while sensitive parts are stored in `credentials.yaml`. | `{}` | [example](#version-20topology-context-bg_domain-example) | ##### \[Version 2.0][Topology Context] `composite_structure` Example diff --git a/docs/features/env-inventory-generation.md b/docs/features/env-inventory-generation.md index e107283c1..d37df4cd3 100644 --- a/docs/features/env-inventory-generation.md +++ b/docs/features/env-inventory-generation.md @@ -8,62 +8,317 @@ - [Goals](#goals) - [Proposed Approach](#proposed-approach) - [Instance Repository Pipeline Parameters](#instance-repository-pipeline-parameters) - - [ENV\_SPECIFIC\_PARAMS](#env_specific_params) - - [ENV\_SPECIFIC\_PARAMS Example](#env_specific_params-example) - - [Generated Environment Inventory Examples](#generated-environment-inventory-examples) + - [`ENV_INVENTORY_CONTENT`](#env_inventory_content) + - [Actions](#actions) + - [Paths by place](#paths-by-place) + - [Processing Model](#processing-model) + - [Validations](#validations) + - [Full `ENV_INVENTORY_CONTENT` Example](#full-env_inventory_content-example) + - [ENV\_INVENTORY\_CONTENT in JSON-in-string format](#env_inventory_content-in-json-in-string-format) + - [`ENV_SPECIFIC_PARAMS`](#env_specific_params) + - [`ENV_SPECIFIC_PARAMS` Example](#env_specific_params-example) + - [Example of Generated Result with `ENV_INVENTORY_CONTENT`](#example-of-generated-result-with-env_inventory_content) + - [Generated Result with `ENV_INVENTORY_CONTENT` (new files)](#generated-result-with-env_inventory_content-new-files) + - [Environment Inventory (`env_definition.yml`)](#environment-inventory-env_definitionyml) + - [Parameter Sets](#parameter-sets) + - [Credentials](#credentials) + - [Resource Profile Overrides](#resource-profile-overrides) + - [Shared Template Variable Files](#shared-template-variable-files) + - [Generated Result when the target file already exists](#generated-result-when-the-target-file-already-exists) + - [env\_definition file already exists](#env_definition-file-already-exists) + - [Existing env\_definition file](#existing-env_definition-file) + - [Input request (ENV\_INVENTORY\_CONTENT)](#input-request-env_inventory_content) + - [Result `env_definition.yml`](#result-env_definitionyml) + - [Parameter Sets file already exists](#parameter-sets-file-already-exists) + - [Existing Parameter Set file](#existing-parameter-set-file) + - [Input request (paramSets)](#input-request-paramsets) + - [Result Parameter Sets](#result-parameter-sets) + - [Example of Generated Result with `ENV_SPECIFIC_PARAMS`](#example-of-generated-result-with-env_specific_params) - [Minimal Environment Inventory](#minimal-environment-inventory) - [Environment Inventory with env-specific parameters](#environment-inventory-with-env-specific-parameters) ## Problem Statements -Current implementations of EnvGene require manual creation of Environment Inventories via working directly with repositories. While external systems can abstract this complexity for their users, EnvGene lacks interface to support such automation for external systems. +Current implementations of EnvGene require manual creation of Environment Inventories via working directly with repositories, and also related Inventory objects such as Parameter Sets, Credentials, Resource Profile Overrides and Shared Template Variable Files. While external systems can abstract this complexity for their users, EnvGene lacks an interface to support such automation for external systems. ### Goals -Develop a interface in EnvGene that enables external systems to create Environment Inventories without direct repository manipulation +Develop an interface in EnvGene that enables external systems to create/replace and delete Environment Inventory and related objects without direct manual repository manipulation, including support for placing files on different levels (site/cluster/env). ## Proposed Approach -It is proposed implementing an EnvGene feature for Environment Inventory generation with a corresponding interface that will allow external systems to create Environment Inventories. +It is proposed to implement an EnvGene feature for Environment Inventory generation with a corresponding interface that will allow external systems to create Environment Inventories. -The external system will initiate Environment Inventory generation by triggering the instance pipeline, passing required variables via the newly introduced [parameters](#instance-repository-pipeline-parameters). The target Environment for Inventory generation is determined by the `ENV_NAMES` attribute. Generating Inventories for multiple Environments in a single pipeline run is not supported. +The external system will initiate Environment Inventory generation by triggering the Instance pipeline, passing required variables via the newly introduced [parameters](#instance-repository-pipeline-parameters). The target Environment for Inventory generation is determined by the `ENV_NAMES` attribute. Generating Inventories for multiple Environments in a single pipeline run **is not supported**. -The solution supports creation of: +The solution supports creation/replace and delete of: -- Environment Inventory -- Environment-specific Parameter Sets -- Credentials - -The created objects are validated according to the corresponding schemes. +- [Environment Inventory](/docs/envgene-configs.md#env_definitionyml) (`env_definition.yml`) +- [Environment-specific Parameter Sets](/docs/envgene-objects.md#environment-specific-parameterset) +- [Shared Credentials](/docs/envgene-objects.md#shared-credentials-file) +- [Resource Profile Overrides](/docs/envgene-objects.md#resource-profile-override) +- [Shared Template Variable Files](/docs/envgene-objects.md#shared-template-variable-files) Generation will occur in a dedicated job within the Instance repository pipeline. The generated Environment Inventory must be reused by other jobs in the same pipeline. In order to be able to generate an Environment Inventory and get an Environment Instance or Effective Set in a single run of the pipeline. To make this possible, it must be executed before any jobs that consume the inventory. -When the inventory already exists, update rules vary depending on parameters. See details in [ENV_SPECIFIC_PARAMS](#env_specific_params) +`ENV_INVENTORY_CONTENT` is the primary way to manage Inventory via pipeline. It allows external systems to create, fully replace and delete `env_definition.yml` and related Inventory objects. The parameter also supports creating files on different levels (`site`, `cluster`, `env`) via the `place` field. + +> **Note** +> If `ENV_TEMPLATE_VERSION` is provided in the instance pipeline, it has higher priority than the template version specified in `env_definition.yml` + +`ENV_SPECIFIC_PARAMS` and `ENV_INVENTORY_INIT` are legacy parameters and are deprecated. They do not cover the full set of Inventory management scenarios, therefore new integrations should use `ENV_INVENTORY_CONTENT`. ### Instance Repository Pipeline Parameters | Parameter | Type | Mandatory | Description | Example | -|-----------|-------------|------|---------|----------| -| `ENV_INVENTORY_INIT` | string | no | If `true`, the new Env Inventory will be generated in the path `/environments///Inventory/env_definition.yml`. If `false` can be updated only | `true` OR `false` | -| `ENV_SPECIFIC_PARAMS` | JSON in string | no | If specified, Env Inventory is updated. See details in [ENV_SPECIFIC_PARAMS](#env_specific_params) | See [example below](#env_specific_params-example) | +| ----------- | ------------- | ------ | --------- | ---------- | +| `ENV_INVENTORY_CONTENT` | JSON in string | no | Allows to create/ replace, delete `env_definition.yml` and related Inventory objects. Must be valid according to [JSON schema](/schemas/env-inventory-content.schema.json). See details in [ENV_INVENTORY_CONTENT](#env_inventory_content) | See [example below](#full-env_inventory_content-example) | +| `ENV_INVENTORY_INIT` | string | no | **Deprecated**. If `true`, the new Environment Inventory will be generated in the path `/environments///Inventory/env_definition.yml`. If `false` can be updated only | `true` OR `false` | +| `ENV_SPECIFIC_PARAMS` | JSON in string | no | **Deprecated**. If specified, Environment Inventory is updated. See details in [ENV_SPECIFIC_PARAMS](#env_specific_params) | See [example below](#env_specific_params-example) | -#### ENV_SPECIFIC_PARAMS +#### `ENV_INVENTORY_CONTENT` | Field | Type | Mandatory | Description | Example | -|-------|-------------|------|---------|----------| +| --- | --- | --- | --- | --- | +| `envDefinition` | object | no | Block that controls `env_definition.yml` | | +| `envDefinition.action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for `env_definition.yml`. See [Actions](#actions) | `create_or_replace` | +| `envDefinition.content` | object | no | Full content of `env_definition.yml`. Must be valid according [schema](/schemas/env-definition.schema.json). See details in [env_definition](/docs/envgene-configs.md#env_definitionyml) | See [example below](#full-env_inventory_content-example) | +| `paramSets` | array | no | Block that controls Parameter Set file operations. | See [example below](#full-env_inventory_content-example) | +| `paramSets[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the target Parameter Set file. See [Actions](#actions) | `create_or_replace` | +| `paramSets[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Parameter Set file is stored. See [Paths by place](#paths-by-place) | `env` | +| `paramSets[].content` | hashmap | no | Parameter Set definition as file content. Must be valid according [schema](/schemas/paramset.schema.json) | See [example below](#full-env_inventory_content-example) | +| `credentials` | array | no | Block that controls Shared Credentials operations | See [example below](#full-env_inventory_content-example) | +| `credentials[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the Shared Credentials file. See [Actions](#actions) | `create_or_replace` | +| `credentials[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Shared Credentials file is stored. See [Paths by place](#paths-by-place) | `site` | +| `credentials[].content` | hashmap | no | Shared Credential as file content. Must be valid according [schema](/schemas/credential.schema.json) | See [example below](#full-env_inventory_content-example) | +| `resourceProfiles` | array | no | List of Resource Profile Override operations | See [example below](#full-env_inventory_content-example) | +| `resourceProfiles[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the Resource Profile Override file. See [Actions](#actions) | `create_or_replace` | +| `resourceProfiles[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Resource Profile Override file is stored. See [Paths by place](#paths-by-place) | `cluster` | +| `resourceProfiles[].content` | hashmap | no | Resource Profile Override as file content. Must be valid according [schema](/schemas/resource-profile.schema.json) | See [example below](#full-env_inventory_content-example) | +| `sharedTemplateVariables` | array | no | Block that controls Shared Template Variable File operations | See [example below](#full-env_inventory_content-example) | +| `sharedTemplateVariables[].action` | enum [`create_or_replace`, `delete`] | yes | Operation mode for the Shared Template Variable File. See [Actions](#actions) | `create_or_replace` | +| `sharedTemplateVariables[].place` | enum[`site`,`cluster`,`env`] | yes | Defines where the Shared Template Variable File is stored. See [Paths by place](#paths-by-place) | `site` | +| `sharedTemplateVariables[].name` | string | yes | Name of the Shared Template Variable File (without extension). The file will be saved as `.yml` | `prod-template-variables` | +| `sharedTemplateVariables[].content` | hashmap | no | Shared Template Variable File content as key-value hashmap. Must NOT be located in a `parameters` directory | See [example below](#full-env_inventory_content-example) | + +##### Actions + +The `action` field defines the operation mode for Inventory objects. + +| Action | Description | +| --- | --- | +| `create_or_replace` | Creates the file if not exist; if the file exists, fully replaces it. All required directories in the path are created automatically if they don't exist | +| `delete` | Deletes the target file if it exists. For `envDefinition`, the entire environment directory `/environments///` is removed. For other object types, only the file is deleted; directories are not removed | + +##### Paths by place + +The pipeline handles Inventory files in the **Instance repository**. +The exact target folder depends on the object type and the `place` value. + +| Object | place=env | place=cluster | place=site | +| --- | --- | --- | --- | +| `envDefinition` | `/environments///Inventory/env_definition.yml` (fixed) | n/a | n/a | +| Parameter Set file | `/environments///Inventory/parameters/.yml` | `/environments//parameters/.yml` | `/environments/parameters/.yml` | +| Shared Credentials file | `/environments///Inventory/credentials/.yml` | `/environments//credentials/.yml` | `/environments/credentials/.yml` | +| Resource Profile Override file | `/environments///Inventory/resource_profiles/.yml` | `/environments//resource_profiles/.yml` | `/environments/resource_profiles/.yml` | +| Shared Template Variable File | `/environments///shared-template-variables/.yml` | `/environments//shared-template-variables/.yml` | `/environments/shared-template-variables/.yml` | + +##### Processing Model + +All operations specified in `ENV_INVENTORY_CONTENT` are processed atomically: either all requested changes are applied successfully, or none of them are applied. If any validation fails or any operation encounters an error, the entire transaction is rolled back and no files are modified. + +The order in which different object types are processed is not guaranteed and may vary. Objects within the same type (e.g., multiple items in `paramSets` array) are also processed in an arbitrary order. + +##### Validations + +Before processing any files, the system performs the following validations: + +**Parameter exclusivity validation:** + +If both `ENV_INVENTORY_CONTENT` and any of `ENV_INVENTORY_INIT` or `ENV_SPECIFIC_PARAMS` are provided, validation fails + +**JSON schema validation:** + +`ENV_INVENTORY_CONTENT` is validated against the [JSON schema](/schemas/env-inventory-content.schema.json) + +If any validation fails, the pipeline stops with a readable validation error in logs and no files are modified. + +##### Full `ENV_INVENTORY_CONTENT` Example + +This example shows how to generate a new Environment Inventory (`env_definition.yml`) and create related objects in the Instance repository: Parameter Sets, Credentials, Resource Profile Overrides, and Shared Template Variable Files. + +```json +{ + "envDefinition": { + "action": "create_or_replace", + "content": { + "inventory": { + "environmentName": "env-1", + "tenantName": "Applications", + "cloudName": "cluster-1", + "description": "Full sample", + "owners": "Qubership team", + "config": { + "updateRPOverrideNameWithEnvName": false, + "updateCredIdsWithEnvName": true + } + }, + "envTemplate": { + "name": "composite-prod", + "artifact": "project-env-template:master_20231024-080204", + "additionalTemplateVariables": { + "ci": { + "CI_PARAM_1": "ci-param-val-1", + "CI_PARAM_2": "ci-param-val-2" + }, + "e2eParameters": { + "E2E_PARAM_1": "e2e-param-val-1", + "E2E_PARAM_2": "e2e-param-val-2" + } + }, + "sharedTemplateVariables": [ + "prod-template-variables", + "sample-cloud-template-variables" + ], + "envSpecificParamsets": { + "bss": [ + "env-specific-bss" + ] + }, + "envSpecificTechnicalParamsets": { + "bss": [ + "env-specific-tech" + ] + }, + "envSpecificE2EParamsets": { + "cloud": [ + "cloud-level-params" + ] + }, + "sharedMasterCredentialFiles": [ + "prod-integration-creds" + ], + "envSpecificResourceProfiles": { + "cloud": [ + "cloud-specific-profile" + ] + } + } + } + }, + "paramSets": [ + { + "action": "create_or_replace", + "place": "env", + "content": { + "version": "", + "name": "env-specific-bss", + "parameters": { + "key": "value" + }, + "applications": [] + } + } + ], + "credentials": [ + { + "action": "create_or_replace", + "place": "site", + "content": { + "prod-integration-creds": { + "type": "", + "data": { + "username": "", + "password": "" + } + } + } + } + ], + "resourceProfiles": [ + { + "action": "create_or_replace", + "place": "cluster", + "content": { + "name": "cloud-specific-profile", + "baseline": "dev", + "description": "", + "applications": [ + { + "name": "core", + "version": "release-20241103.225817", + "sd": "", + "services": [ + { + "name": "operator", + "parameters": [ + { + "name": "GATEWAY_MEMORY_LIMIT", + "value": "96Mi" + }, + { + "name": "GATEWAY_CPU_REQUEST", + "value": "50m" + } + ] + } + ] + } + ], + "version": 0 + } + } + ], + "sharedTemplateVariables": [ + { + "action": "create_or_replace", + "place": "site", + "name": "prod-template-variables", + "content": { + "TEMPLATE_VAR_1": "prod-value-1", + "TEMPLATE_VAR_2": "prod-value-2", + "nested": { + "key1": "nested-prod-value-1", + "key2": "nested-prod-value-2" + } + } + }, + { + "action": "create_or_replace", + "place": "cluster", + "name": "sample-cloud-template-variables", + "content": { + "CLOUD_VAR_1": "cloud-value-1", + "CLOUD_VAR_2": "cloud-value-2" + } + } + ] +} +``` + +##### ENV_INVENTORY_CONTENT in JSON-in-string format + +```json +"{\"envDefinition\":{\"action\":\"create_or_replace\",\"content\":{\"inventory\":{\"environmentName\":\"env-1\",\"tenantName\":\"Applications\",\"cloudName\":\"cluster-1\",\"description\":\"Fullsample\",\"owners\":\"Qubershipteam\",\"config\":{\"updateRPOverrideNameWithEnvName\":false,\"updateCredIdsWithEnvName\":true}},\"envTemplate\":{\"name\":\"composite-prod\",\"artifact\":\"project-env-template:master_20231024-080204\",\"additionalTemplateVariables\":{\"ci\":{\"CI_PARAM_1\":\"ci-param-val-1\",\"CI_PARAM_2\":\"ci-param-val-2\"},\"e2eParameters\":{\"E2E_PARAM_1\":\"e2e-param-val-1\",\"E2E_PARAM_2\":\"e2e-param-val-2\"}},\"sharedTemplateVariables\":[\"prod-template-variables\",\"sample-cloud-template-variables\"],\"envSpecificParamsets\":{\"bss\":[\"env-specific-bss\"]},\"envSpecificTechnicalParamsets\":{\"bss\":[\"env-specific-tech\"]},\"envSpecificE2EParamsets\":{\"cloud\":[\"cloud-level-params\"]},\"sharedMasterCredentialFiles\":[\"prod-integration-creds\"],\"envSpecificResourceProfiles\":{\"cloud\":[\"cloud-specific-profile\"]}}}},\"paramSets\":[{\"action\":\"create_or_replace\",\"place\":\"env\",\"content\":{\"version\":\"\",\"name\":\"env-specific-bss\",\"parameters\":{\"key\":\"value\"},\"applications\":[]}}],\"credentials\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"content\":{\"prod-integration-creds\":{\"type\":\"\",\"data\":{\"username\":\"\",\"password\":\"\"}}}}],\"resourceProfiles\":[{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"content\":{\"name\":\"cloud-specific-profile\",\"baseline\":\"dev\",\"description\":\"\",\"applications\":[{\"name\":\"core\",\"version\":\"release-20241103.225817\",\"sd\":\"\",\"services\":[{\"name\":\"operator\",\"parameters\":[{\"name\":\"GATEWAY_MEMORY_LIMIT\",\"value\":\"96Mi\"},{\"name\":\"GATEWAY_CPU_REQUEST\",\"value\":\"50m\"}]}]}],\"version\":0}}],\"sharedTemplateVariables\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"name\":\"prod-template-variables\",\"content\":{\"TEMPLATE_VAR_1\":\"prod-value-1\",\"TEMPLATE_VAR_2\":\"prod-value-2\",\"nested\":{\"key1\":\"nested-prod-value-1\",\"key2\":\"nested-prod-value-2\"}}},{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"name\":\"sample-cloud-template-variables\",\"content\":{\"CLOUD_VAR_1\":\"cloud-value-1\",\"CLOUD_VAR_2\":\"cloud-value-2\"}}]}" +``` + +#### `ENV_SPECIFIC_PARAMS` + +| Field | Type | Mandatory | Description | Example | +| ------- | ------------- | ------ | --------- | ---------- | | `clusterParams` | hashmap | no | Cluster connection parameters | None | -| `clusterParams.clusterEndpoint` | string | no | System **overrides** the value of `inventory.clusterUrl` in corresponding Env Inventory | `https://api.cluster.example.com:6443` | -| `clusterParams.clusterToken` | string | no | System **adds** Credential in the `/environments///Credentials/inventory_generation_creds.yml`. If Credential already exists, the value will **not be overridden**. System also creates an association with the credential file in corresponding Env Inventory via `envTemplate.sharedMasterCredentialFiles` | None | -| `additionalTemplateVariables` | hashmap | no | System **merges** the value into `envTemplate.additionalTemplateVariables` in corresponding Env Inventory | `{"keyA": "valueA", "keyB": "valueB"}` | -| `cloudName` | string | no | System **overrides** the value of `inventory.cloudName` in corresponding Env Inventory | `cloud01` | -| `tenantName` | string | no | System **overrides** the value of `inventory.tenantName` in corresponding Env Inventory | `Application` | -| `deployer` | string | no | System **overrides** the value of `inventory.deployer` in corresponding Env Inventory | `abstract-CMDB-1` | -| `envSpecificParamsets` | hashmap | no | System **merges** the value into envTemplate.envSpecificParamsets in corresponding Env Inventory | See [example](#env_specific_params-example) | -| `paramsets` | hashmap | no | System creates Parameter Set file for each first level key in the path `environments///Inventory/parameters/KEY-NAME.yml`. If Parameter Set already exists, the value will be **overridden** | See [example](#env_specific_params-example) | -| `credentials` | hashmap | no | System **adds** Credential object for each first level key in the `/environments///credentials/inventory_generation_creds.yml`. If Credential already exists, the value will be **overridden**. System also creates an association with the credential file in corresponding Env Inventory via `envTemplate.sharedMasterCredentialFiles` | See [example](#env_specific_params-example) | - -##### ENV_SPECIFIC_PARAMS Example +| `clusterParams.clusterEndpoint` | string | no | System **overrides** the value of `inventory.clusterUrl` in corresponding Environment Inventory | `https://api.cluster.example.com:6443` | +| `clusterParams.clusterToken` | string | no | System **adds** Credential in the `/environments///Inventory/credentials/inventory_generation_creds.yml`. If Credential already exists, the value will **not be overridden**. System also creates an association with the credential file in corresponding Environment Inventory via `envTemplate.sharedMasterCredentialFiles` | None | +| `additionalTemplateVariables` | hashmap | no | System **merges** the value into `envTemplate.additionalTemplateVariables` in corresponding Environment Inventory | `{"keyA": "valueA", "keyB": "valueB"}` | +| `cloudName` | string | no | System **overrides** the value of `inventory.cloudName` in corresponding Environment Inventory | `cloud01` | +| `tenantName` | string | no | System **overrides** the value of `inventory.tenantName` in corresponding Environment Inventory | `Application` | +| `deployer` | string | no | System **overrides** the value of `inventory.deployer` in corresponding Environment Inventory | `abstract-CMDB-1` | +| `envSpecificParamsets` | hashmap | no | System **merges** the value into `envTemplate.envSpecificParamsets` in corresponding Environment Inventory | See [example](#env_specific_params-example) | +| `paramsets` | hashmap | no | System creates Parameter Set file for each first level key in the path `/environments///Inventory/parameters/KEY-NAME.yml`. If Parameter Set already exists, the value will be **overridden** | See [example](#env_specific_params-example) | +| `credentials` | hashmap | no | System **adds** Credential object for each first level key in the `/environments///Inventory/credentials/inventory_generation_creds.yml`. If Credential already exists, the value will be **overridden**. System also creates an association with the credential file in corresponding Environment Inventory via `envTemplate.sharedMasterCredentialFiles` | See [example](#env_specific_params-example) | + +##### `ENV_SPECIFIC_PARAMS` Example ```json { @@ -128,12 +383,261 @@ When the inventory already exists, update rules vary depending on parameters. Se } ``` -### Generated Environment Inventory Examples +### Example of Generated Result with `ENV_INVENTORY_CONTENT` + +#### Generated Result with `ENV_INVENTORY_CONTENT` (new files) + +##### Environment Inventory (`env_definition.yml`) + +**Result:** `env_definition.yml` is generated from envDefinition.content. + +```yaml +# /environments///Inventory/env_definition.yml + +inventory: + environmentName: "env-1" + tenantName: "Applications" + cloudName: "cluster-1" + description: "Full sample" + owners: "Qubership team" + config: + updateRPOverrideNameWithEnvName: false + updateCredIdsWithEnvName: true + +envTemplate: + name: "composite-prod" + artifact: "project-env-template:master_20231024-080204" + + additionalTemplateVariables: + ci: + CI_PARAM_1: "ci-param-val-1" + CI_PARAM_2: "ci-param-val-2" + e2eParameters: + E2E_PARAM_1: "e2e-param-val-1" + E2E_PARAM_2: "e2e-param-val-2" + + sharedTemplateVariables: + - "prod-template-variables" + - "sample-cloud-template-variables" + + envSpecificParamsets: + bss: + - "env-specific-bss" + + envSpecificTechnicalParamsets: + bss: + - "env-specific-tech" + + envSpecificE2EParamsets: + cloud: + - "cloud-level-params" + + sharedMasterCredentialFiles: + - "prod-integration-creds" + + envSpecificResourceProfiles: + cloud: + - "cloud-specific-profile" +``` + +##### Parameter Sets + +**Result**: a Parameter Set file is generated from paramSets[].content and stored based on paramSets[].place. + +```yaml +# /environments///Inventory/parameters/env-specific-bss.yml + +version: "" +name: "env-specific-bss" +parameters: + key: "value" +applications: [] +``` + +##### Credentials + +**Result**: a Credentials file is generated from credentials[].content and stored based on credentials[].place. + +```yaml +# /environments/credentials/prod-integration-creds.yml + +prod-integration-creds: + type: + data: + username: "" + password: "" + +``` + +##### Resource Profile Overrides + +**Result**: a Resource Profile Override file is generated from resourceProfiles[].content and stored based on resourceProfiles[].place. + +```yaml +# /environments//Inventory/resource_profiles/cloud-specific-profile.yml + +name: "cloud-specific-profile" +baseline: "dev" +description: "" +applications: +- name: "core" + version: "release-20241103.225817" + sd: "" + services: + - name: "operator" + parameters: + - name: "GATEWAY_MEMORY_LIMIT" + value: "96Mi" + - name: "GATEWAY_CPU_REQUEST" + value: "50m" +``` + +##### Shared Template Variable Files + +**Result**: a Shared Template Variable File is generated from sharedTemplateVariables[].content and stored based on sharedTemplateVariables[].place. + +```yaml +# /environments/shared-template-variables/prod-template-variables.yml + +TEMPLATE_VAR_1: "prod-value-1" +TEMPLATE_VAR_2: "prod-value-2" +nested: + key1: "nested-prod-value-1" + key2: "nested-prod-value-2" +``` + +```yaml +# /environments//shared-template-variables/sample-cloud-template-variables.yml + +CLOUD_VAR_1: "cloud-value-1" +CLOUD_VAR_2: "cloud-value-2" +``` + +#### Generated Result when the target file already exists + +##### env_definition file already exists + +###### Existing env_definition file + +```yaml +# /environments///Inventory/env_definition.yml +inventory: + environmentName: "env-1" + tenantName: "Applications" + cloudName: "cluster-1" + +envTemplate: + name: "composite-prod" + artifact: "project-env-template:old" + envSpecificParamsets: + bss: + - "env-specific-bss" +``` + +###### Input request (ENV_INVENTORY_CONTENT) + +```json +{ + "envDefinition": { + "action": "create_or_replace", + "content": { + "inventory": { + "environmentName": "env-1", + "tenantName": "Applications", + "cloudName": "cluster-1", + "description": "Updated description", + "config": { + "updateCredIdsWithEnvName": true + } + }, + "envTemplate": { + "name": "composite-prod", + "artifact": "project-env-template:new", + "envSpecificE2EParamsets": { + "cloud": [ + "cloud-level-params" + ] + }, + "sharedMasterCredentialFiles": [ + "prod-integration-creds" + ] + } + } + } +} +``` + +###### Result `env_definition.yml` + +```yaml +# /environments///Inventory/env_definition.yml +inventory: + environmentName: "env-1" + tenantName: "Applications" + cloudName: "cluster-1" + description: "Updated description" + config: + updateCredIdsWithEnvName: true + +envTemplate: + name: "composite-prod" + artifact: "project-env-template:new" + envSpecificE2EParamsets: + cloud: + - "cloud-level-params" + sharedMasterCredentialFiles: + - "prod-integration-creds" +``` + +##### Parameter Sets file already exists + +###### Existing Parameter Set file + +```yaml +# /environments///Inventory/parameters/env-specific-bss.yml +name: "env-specific-bss" +parameters: + featureFlag: "false" +applications: [] +``` + +###### Input request (paramSets) + +```json +{ + "paramSets": [ + { + "action": "create_or_replace", + "place": "env", + "content": { + "version": "1.1", + "name": "env-specific-bss", + "parameters": { + "featureFlag": "true" + }, + "applications": [] + } + } + ] +} +``` + +###### Result Parameter Sets + +```yaml +# /environments///Inventory/parameters/env-specific-bss.yml +name: "env-specific-bss" +parameters: + featureFlag: "true" +applications: [] +``` + +### Example of Generated Result with `ENV_SPECIFIC_PARAMS` #### Minimal Environment Inventory ```yaml -# /environments///Inventory/env_definition.yml +# /environments///Inventory/env_definition.yml inventory: environmentName: clusterUrl: @@ -145,7 +649,7 @@ envTemplate: #### Environment Inventory with env-specific parameters ```yaml -# /environments///Inventory/env_definition.yml +# /environments///Inventory/env_definition.yml inventory: environmentName: clusterUrl: @@ -161,7 +665,7 @@ envTemplate: ``` ```yaml -# /environments///Credentials/credentials.yml +# /environments///Credentials/credentials.yml cloud-admin-token: type: "secret" data: @@ -169,7 +673,7 @@ cloud-admin-token: ``` ```yaml -# environments///Inventory/parameters/paramsetA.yml +# /environments///Inventory/parameters/paramsetA.yml paramsetA: version: name: @@ -182,7 +686,7 @@ paramsetA: ``` ```yaml -# environments///Inventory/parameters/paramsetB.yml +# /environments///Inventory/parameters/paramsetB.yml paramsetB: version: name: @@ -192,7 +696,7 @@ paramsetB: ``` ```yaml -# environments///Inventory/credentials/inventory_generation_creds.yml +# /environments///Inventory/credentials/inventory_generation_creds.yml credX: type: data: diff --git a/docs/instance-pipeline-parameters.md b/docs/instance-pipeline-parameters.md index 8a6501fba..308f57f41 100644 --- a/docs/instance-pipeline-parameters.md +++ b/docs/instance-pipeline-parameters.md @@ -9,11 +9,9 @@ - [`CMDB_IMPORT`](#cmdb_import) - [`DEPLOYMENT_TICKET_ID`](#deployment_ticket_id) - [`ENV_TEMPLATE_VERSION`](#env_template_version) - - [`ENV_TEMPLATE_VERSION_ORIGIN`](#env_template_version_origin) - - [`ENV_TEMPLATE_VERSION_PEER`](#env_template_version_peer) - - [`ENV_INVENTORY_INIT`](#env_inventory_init) - - [`ENV_TEMPLATE_NAME`](#env_template_name) + - [`ENV_TEMPLATE_VERSION_UPDATE_MODE`](#env_template_version_update_mode) - [`ENV_SPECIFIC_PARAMS`](#env_specific_params) + - [`ENV_INVENTORY_CONTENT`](#env_inventory_content) - [`GENERATE_EFFECTIVE_SET`](#generate_effective_set) - [`EFFECTIVE_SET_CONFIG`](#effective_set_config) - [`APP_REG_DEFS_JOB`](#app_reg_defs_job) @@ -28,9 +26,6 @@ - [`CRED_ROTATION_PAYLOAD`](#cred_rotation_payload) - [Affected Parameters and Troubleshooting](#affected-parameters-and-troubleshooting) - [`CRED_ROTATION_FORCE`](#cred_rotation_force) - - [`SD_REPO_MERGE_MODE`](#sd_repo_merge_mode) - - [`NS_BUILD_FILTER`](#ns_build_filter) - - [`GITHUB_PIPELINE_API_INPUT`](#github_pipeline_api_input) - [`GH_ADDITIONAL_PARAMS`](#gh_additional_params) - [`BG_MANAGE`](#bg_manage) - [`BG_STATE`](#bg_state) @@ -118,6 +113,30 @@ This parameter serves as a configuration for an extension point. Integration wit **Example**: `env-template:v1.2.3` +### `ENV_TEMPLATE_VERSION_UPDATE_MODE` + +**Description**: Controls how ENV_TEMPLATE_VERSION is applied during the pipeline run. + +**Allowed values**: + +- `PERSISTENT` (default) + Applies the standard behavior: the pipeline updates the template version in Environment Inventory by updating `envTemplate.artifact` (or `envTemplate.templateArtifact.artifact.version`) in `env_definition.yml`. + +- `TEMPORARY` + Applies `ENV_TEMPLATE_VERSION` **only for the current pipeline execution** and **does not** update `envTemplate.artifact` (or `envTemplate.templateArtifact.artifact.version`) in `env_definition.yml`. + The pipeline updates `generatedVersions.generateEnvironmentLatestVersion` in `env_definition.yml` to reflect the template artifact version that was actually applied in this run, for example: + + ```yaml + # env_definition.yml + generatedVersions: + generateEnvironmentLatestVersion: "template-project:feature-diis1125-20251125.045717-2" + +**Default Value**: `PERSISTENT` + +**Mandatory**: No + +**Example**: `PERSISTENT` + ### `ENV_TEMPLATE_VERSION_ORIGIN` **Description**: If provided, system updates the Blue-Green origin template artifact version in the Environment Inventory. System overrides `envTemplate.bgNsArtifacts.origin` at `/environments//Inventory/env_definition.yml` @@ -178,6 +197,8 @@ envTemplate: **Description**: Specifies Environment Inventory and env-specific parameters. This is can used together with `ENV_INVENTORY_INIT`. **JSON in string** format. See details in [Environment Inventory Generation](/docs/features/env-inventory-generation.md) +**Note:** This parameter is deprecated and will be removed in future releases. Use `ENV_INVENTORY_CONTENT` instead. + **Default Value**: None **Mandatory**: No @@ -188,6 +209,25 @@ envTemplate: '{"clusterParams":{"clusterEndpoint":"","clusterToken":""},"additionalTemplateVariables":{"":""},"cloudName":"","envSpecificParamsets":{"":["paramsetA"],"cloud":["paramsetB"]},"paramsets":{"paramsetA":{"version":"","name":"","parameters":{"":""},"applications":[{"appName":"","parameters":{"":""}}]},"paramsetB":{"version":"","name":"","parameters":{"":""},"applications":[]}},"credentials":{"credX":{"type":"","data":{"username":"","password":""}},"credY":{"type":"","data":{"secret":""}}}}' ``` +### `ENV_INVENTORY_CONTENT` + +**Description**: + +Provides the Environment Inventory and related artifacts to be created or updated. +It allows external systems to manage `env_definition.yml` and additional files paramsets, credentials, resource profiles without manual changes in the Instance repository. + +See details in Environment Inventory Generation feature documentation [Environment Inventory Generation](/docs/features/env-inventory-generation.md) + +**Default Value**: None + +**Mandatory**: No + +**Example in string format**: + +```json +"{\"envDefinition\":{\"action\":\"create_or_replace\",\"content\":{\"inventory\":{\"environmentName\":\"env-1\",\"tenantName\":\"Applications\",\"cloudName\":\"cluster-1\",\"description\":\"Fullsample\",\"owners\":\"Qubershipteam\",\"config\":{\"updateRPOverrideNameWithEnvName\":false,\"updateCredIdsWithEnvName\":true}},\"envTemplate\":{\"name\":\"composite-prod\",\"artifact\":\"project-env-template:master_20231024-080204\",\"additionalTemplateVariables\":{\"ci\":{\"CI_PARAM_1\":\"ci-param-val-1\",\"CI_PARAM_2\":\"ci-param-val-2\"},\"e2eParameters\":{\"E2E_PARAM_1\":\"e2e-param-val-1\",\"E2E_PARAM_2\":\"e2e-param-val-2\"}},\"sharedTemplateVariables\":[\"prod-template-variables\",\"sample-cloud-template-variables\"],\"envSpecificParamsets\":{\"bss\":[\"env-specific-bss\"]},\"envSpecificTechnicalParamsets\":{\"bss\":[\"env-specific-tech\"]},\"envSpecificE2EParamsets\":{\"cloud\":[\"cloud-level-params\"]},\"sharedMasterCredentialFiles\":[\"prod-integration-creds\"],\"envSpecificResourceProfiles\":{\"cloud\":[\"cloud-specific-profile\"]}}}},\"paramsets\":[{\"action\":\"create_or_replace\",\"place\":\"env\",\"content\":{\"version\":\"\",\"name\":\"env-specific-bss\",\"parameters\":{\"key\":\"value\"},\"applications\":[]}}],\"credentials\":[{\"action\":\"create_or_replace\",\"place\":\"site\",\"content\":{\"prod-integration-creds\":{\"type\":\"\",\"data\":{\"username\":\"\",\"password\":\"\"}}}}],\"resourceProfiles\":[{\"action\":\"create_or_replace\",\"place\":\"cluster\",\"content\":{\"name\":\"cloud-specific-profile\",\"baseline\":\"dev\",\"description\":\"\",\"applications\":[{\"name\":\"core\",\"version\":\"release-20241103.225817\",\"sd\":\"\",\"services\":[{\"name\":\"operator\",\"parameters\":[{\"name\":\"GATEWAY_MEMORY_LIMIT\",\"value\":\"96Mi\"},{\"name\":\"GATEWAY_CPU_REQUEST\",\"value\":\"50m\"}]}]}],\"version\":0}}]}" +``` + ### `GENERATE_EFFECTIVE_SET` **Description**: Feature flag. Valid values ​​are `true` or `false`. @@ -209,6 +249,7 @@ If `true`: version: effective_set_expiry: app_chart_validation: +enable_traceability: contexts: pipeline: consumers: @@ -217,15 +258,16 @@ contexts: schema: ``` -| Attribute | Mandatory | Description | Default | Example | -|---|---|---|---|---| -| **version** | Optional | The version of the effective set to be generated. Available options are `v1.0` and `v2.0`. EnvGene uses `--effective-set-version` to pass this attribute to the Calculator CLI. | `v1.0` | `v2.0` | -| **app_chart_validation** | Optional | [App chart validation](/docs/features/calculator-cli.md#version-20-app-chart-validation) feature flag. This validation checks whether all applications in the solution for which the effective set is being calculated are built using the app chart model. If at least one is not, the calculation fails. If `true`: validation is performed, if `false`: validation is skipped | `true` | `false` | -| **effective_set_expiry** | Optional | The duration for which the effective set (stored as a job artifact) will remain available for download. Envgene passes this value unchanged to: 1) The `retention-days` job attribute in case of GitHub pipeline. 2) The `expire_in` job attribute in case of GitLab pipeline. The exact syntax and constraints differ between platforms. Refer to the GitHub and GitLab documentation for details. | GitLab: `1 hours`, GitHub: `1` (day) | GitLab: `2 hours`, GitHub: `2` | -| **contexts.pipeline.consumers** | Optional | Each entry in this list adds a [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context) to the Effective Set. EnvGene passes the path to the corresponding JSON schema file to the Calculator command-line tool using the `--pipeline-consumer-specific-schema-path` argument. Each list element is passed as a separate argument. | None | None | -| **contexts.pipeline.consumers[].name** | Mandatory | The name of the [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context). If used without `contexts.pipeline.consumers[].schema`, the component must be pre-registered in EnvGene | None | `dcl` | -| **contexts.pipeline.consumers[].version** | Mandatory | The version of the [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context). If used without `contexts.pipeline.consumers[].schema`, the component must be pre-registered in EnvGene. | None | `v1.0`| -| **contexts.pipeline.consumers[].schema** | Optional | The content of the consumer-specific pipeline context component JSON schema transformed into a string. It is used to generate a consumer-specific pipeline context for a consumer not registered in EnvGene. EnvGene saves the value as a JSON file with the name `-.schema.json` and passes the path to it to the Calculator command-line tool via `--pipeline-consumer-specific-schema-path` attribute. The schema obtained in this way is not saved between pipeline runs and must be passed for each run. | None | [consumer-v1.0.json](/examples/consumer-v1.0.json) | +| Attribute | Mandatory | Description | Default | Example | +|-------------------------------------------|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|----------------------------------------------------| +| **version** | Optional | The version of the effective set to be generated. Available options are `v1.0` and `v2.0`. EnvGene uses `--effective-set-version` to pass this attribute to the Calculator CLI. | `v2.0` | `v2.0` | +| **app_chart_validation** | Optional | [App chart validation](/docs/features/calculator-cli.md#version-20-app-chart-validation) feature flag. This validation checks whether all applications in the solution for which the effective set is being calculated are built using the app chart model. If at least one is not, the calculation fails. If `true`: validation is performed, if `false`: validation is skipped | `true` | `false` | +| **effective_set_expiry** | Optional | The duration for which the effective set (stored as a job artifact) will remain available for download. Envgene passes this value unchanged to: 1) The `retention-days` job attribute in case of GitHub pipeline. 2) The `expire_in` job attribute in case of GitLab pipeline. The exact syntax and constraints differ between platforms. Refer to the GitHub and GitLab documentation for details. | GitLab: `1 hours`, GitHub: `1` (day) | GitLab: `2 hours`, GitHub: `2` | +| **enable_traceability** | Optional | Feature flag that enables traceability functionality in the effective set generation. When set to `true`, the Calculator CLI will include additional traceability information in the generated effective set, allowing tracking of parameter sources and transformations. When set to `false`, traceability information is not included. | `false` | `true` | +| **contexts.pipeline.consumers** | Optional | Each entry in this list adds a [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context) to the Effective Set. EnvGene passes the path to the corresponding JSON schema file to the Calculator command-line tool using the `--pipeline-consumer-specific-schema-path` argument. Each list element is passed as a separate argument. | None | None | +| **contexts.pipeline.consumers[].name** | Mandatory | The name of the [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context). If used without `contexts.pipeline.consumers[].schema`, the component must be pre-registered in EnvGene | None | `dcl` | +| **contexts.pipeline.consumers[].version** | Mandatory | The version of the [consumer-specific pipeline context component](/docs/features/calculator-cli.md#version-20-pipeline-parameter-context). If used without `contexts.pipeline.consumers[].schema`, the component must be pre-registered in EnvGene. | None | `v1.0` | +| **contexts.pipeline.consumers[].schema** | Optional | The content of the consumer-specific pipeline context component JSON schema transformed into a string. It is used to generate a consumer-specific pipeline context for a consumer not registered in EnvGene. EnvGene saves the value as a JSON file with the name `-.schema.json` and passes the path to it to the Calculator command-line tool via `--pipeline-consumer-specific-schema-path` attribute. The schema obtained in this way is not saved between pipeline runs and must be passed for each run. | None | [consumer-v1.0.json](/examples/consumer-v1.0.json) | Registered component JSON schemas are stored in the EnvGene Docker image as JSON files named: `-.schema.json` diff --git a/docs/use-cases/environment-instance-generation.md b/docs/use-cases/environment-instance-generation.md index 072e79f22..43c91f22d 100644 --- a/docs/use-cases/environment-instance-generation.md +++ b/docs/use-cases/environment-instance-generation.md @@ -363,7 +363,7 @@ Instance pipeline (GitLab or GitHub) is started with parameters: **Results:** 1. All Namespaces are rendered using `project-env-template:v1.2.3` -2. All other objects (Tenant, Cloud, Applications, etc.) are rendered using `project-env-template:v1.2.3` +2. All other objects are rendered using `project-env-template:v1.2.3` ### UC-EIG-TA-2: Environment Instance Generation with `artifact` and `bgNsArtifacts` and BG Domain diff --git a/docs/use-cases/environment-inventory-generation.md b/docs/use-cases/environment-inventory-generation.md new file mode 100644 index 000000000..e3ac0ba57 --- /dev/null +++ b/docs/use-cases/environment-inventory-generation.md @@ -0,0 +1,839 @@ +# Environment Inventory Generation Use Cases + +## Table of Contents + +- [Environment Inventory Generation Use Cases](#environment-inventory-generation-use-cases) + - [Table of Contents](#table-of-contents) + - [Overview](#overview) + - [Environment Inventory: env\_definition.yml](#environment-inventory-env_definitionyml) + - [UC-EINV-ED-1: Create `env_definition.yml` (`create_or_replace`, file does not exist)](#uc-einv-ed-1-create-env_definitionyml-create_or_replace-file-does-not-exist) + - [UC-EINV-ED-2: Replace `env_definition.yml` (`create_or_replace`, file exists)](#uc-einv-ed-2-replace-env_definitionyml-create_or_replace-file-exists) + - [UC-EINV-ED-3: Delete `env_definition.yml`](#uc-einv-ed-3-delete-env_definitionyml) + - [Environment Inventory: Paramsets](#environment-inventory-paramsets) + - [UC-EINV-PS-1: Create paramset file (`create_or_replace`, file does not exist)](#uc-einv-ps-1-create-paramset-file-create_or_replace-file-does-not-exist) + - [UC-EINV-PS-2: Replace paramset file (`create_or_replace`, file exists)](#uc-einv-ps-2-replace-paramset-file-create_or_replace-file-exists) + - [UC-EINV-PS-3: Delete paramSet file](#uc-einv-ps-3-delete-paramset-file) + - [Environment Inventory: Credentials](#environment-inventory-credentials) + - [UC-EINV-CR-1: Create credentials file (`create_or_replace`, file does not exist)](#uc-einv-cr-1-create-credentials-file-create_or_replace-file-does-not-exist) + - [UC-EINV-CR-2: Replace credentials file (`create_or_replace`, file exists)](#uc-einv-cr-2-replace-credentials-file-create_or_replace-file-exists) + - [UC-EINV-CR-3: Delete credentials file](#uc-einv-cr-3-delete-credentials-file) + - [Environment Inventory: Resource Profile Overrides](#environment-inventory-resource-profile-overrides) + - [UC-EINV-RP-1: Create resource profile override file (`create_or_replace`, file does not exist)](#uc-einv-rp-1-create-resource-profile-override-file-create_or_replace-file-does-not-exist) + - [UC-EINV-RP-2: Replace resource profile override file (`create_or_replace`, file exists)](#uc-einv-rp-2-replace-resource-profile-override-file-create_or_replace-file-exists) + - [UC-EINV-RP-3: Delete resource profile override file](#uc-einv-rp-3-delete-resource-profile-override-file) + - [Environment Inventory: Shared Template Variable Files](#environment-inventory-shared-template-variable-files) + - [UC-EINV-STV-1: Create Shared Template Variable file (`create_or_replace`, file does not exist)](#uc-einv-stv-1-create-shared-template-variable-file-create_or_replace-file-does-not-exist) + - [UC-EINV-STV-2: Replace Shared Template Variable file (create\_or\_replace, file exists)](#uc-einv-stv-2-replace-shared-template-variable-file-create_or_replace-file-exists) + - [UC-EINV-STV-3: Delete Shared Template Variable file](#uc-einv-stv-3-delete-shared-template-variable-file) + - [UC-EINV-AT-ALL-1: Rollback all Inventory changes if any operation fails (negative, atomic processing)](#uc-einv-at-all-1-rollback-all-inventory-changes-if-any-operation-fails-negative-atomic-processing) + - [Template Version Update](#template-version-update) + - [UC-EINV-TV-1: Apply `ENV_TEMPLATE_VERSION` (`PERSISTENT` vs `TEMPORARY`)](#uc-einv-tv-1-apply-env_template_version-persistent-vs-temporary) + +--- + +## Overview + +This document describes use cases for **Environment Inventory Generation** — creating or replacing `env_definition.yml`, `paramsets`, `resource_profiles`, and `credentials` using `ENV_INVENTORY_CONTENT`, as well as template version update in `PERSISTENT` and `TEMPORARY` modes. + +> **Note (template version priority):** +> If `ENV_TEMPLATE_VERSION` is passed to the Instance pipeline, it has **higher priority** than the template version specified in `env_definition.yml` (`envDefinition.content.envTemplate.*`). + +--- + +## Environment Inventory: env_definition.yml + +### UC-EINV-ED-1: Create `env_definition.yml` (`create_or_replace`, file does not exist) + +**Pre-requisites:** + +1. The Environment Inventory file does not exist: + - `/environments///Inventory/env_definition.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) +- `ENV_TEMPLATE_VERSION: ` (optional; if provided, it has higher priority than the version from `envDefinition.content.envTemplate.*`) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates `envDefinition` against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `envDefinition.action == create_or_replace` + - `envDefinition.content` is present + 3. Validates `envDefinition.content` against `env_definition.yml` schema: + - [`/docs/envgene-configs.md#env_definitionyml`](/docs/envgene-configs.md#env_definitionyml) + 4. Resolves target path: + - `/environments///Inventory/env_definition.yml` + 5. Creates `Inventory/` directory if missing. + 6. Creates `env_definition.yml` using `envDefinition.content`. + 7. If `ENV_TEMPLATE_VERSION` is provided, applies it as the template version (higher priority). +2. The `git_commit` job runs: + 1. Commits created files into the Instance repository. + +**Results:** + +1. The file is created: + - `/environments///Inventory/env_definition.yml` +2. If `ENV_TEMPLATE_VERSION` is provided, it overrides the version from `envDefinition.content.envTemplate.*`. +3. Changes are committed. + +--- + +### UC-EINV-ED-2: Replace `env_definition.yml` (`create_or_replace`, file exists) + +**Pre-requisites:** + +1. The Environment Inventory file exists: + - `/environments///Inventory/env_definition.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) +- `ENV_TEMPLATE_VERSION: ` (optional; if provided, it has higher priority than the version from `envDefinition.content.envTemplate.*`) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates `envDefinition` against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `envDefinition.action == create_or_replace` + - `envDefinition.content` is present + 3. Validates `envDefinition.content` against `env_definition.yml` schema: + - [`/docs/envgene-configs.md#env_definitionyml`](/docs/envgene-configs.md#env_definitionyml) + 4. Resolves target path: + - `/environments///Inventory/env_definition.yml` + 5. Replaces `env_definition.yml` using `envDefinition.content` (fully overwrites the file). + 6. If `ENV_TEMPLATE_VERSION` is provided, applies it as the template version (higher priority). +2. The `git_commit` job runs: + 1. Commits updated files into the Instance repository. + +**Results:** + +1. The file is replaced (fully overwritten): + - `/environments///Inventory/env_definition.yml` +2. If `ENV_TEMPLATE_VERSION` is provided, it overrides the version from `envDefinition.content.envTemplate.*`. +3. Changes are committed. + +--- + +### UC-EINV-ED-3: Delete `env_definition.yml` + +**Pre-requisites:** + +1. The Environment Inventory file exists: + - `/environments///Inventory/env_definition.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates `envDefinition` against the request schema: + - `envDefinition.action == delete` + - `envDefinition.content` is not required + 3. Resolves target environment folder: + - `/environments///` + 4. Deletes the entire environment directory: + - `/environments///` + +2. The `git_commit` job runs: + 1. Commits repository changes into the Instance repository. + +**Results:** + +1. The environment directory is removed: + - `/environments///` + +2. `env_definition.yml` is removed as part of the environment directory deletion (if it existed): + - `/environments///Inventory/env_definition.yml` + +3. All Inventory-related files under the environment are removed as part of the environment directory deletion (if they existed): + - `/environments///Inventory/parameters/*` + - `/environments///Inventory/credentials/*` + - `/environments///Inventory/resource_profiles/*` + - `/environments///shared-template-variables/*` +4. Changes are committed. + +--- + +## Environment Inventory: Paramsets + +### UC-EINV-PS-1: Create paramset file (`create_or_replace`, file does not exist) + +**Pre-requisites:** + +1. The target paramset file does not exist (for the resolved `place` and `content.name`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `paramsets[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content.name: ` +- `content` is a valid Paramset file content + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `paramsets[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`: + - `place=env` → `/environments///Inventory/parameters/.yml` + - `place=cluster` → `/environments//Inventory/parameters/.yml` + - `place=site` → `/environments/Inventory/parameters/.yml` + 5. Creates `parameters/` directory if missing. + 6. Creates the paramset file using `content` (create-or-replace semantics; in this UC the file is expected to be missing). +2. The `git_commit` job runs: + 1. Commits created files into the Instance repository. + +**Results:** + +1. Paramset file is created at the resolved path. +2. File content matches `paramsets[].content`. +3. Changes are committed. + +--- + +### UC-EINV-PS-2: Replace paramset file (`create_or_replace`, file exists) + +**Pre-requisites:** + +1. The target paramset file exists (for the resolved `place` and `content.name`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `paramsets[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content.name: ` +- `content` is a valid Paramset file content + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `paramsets[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place == { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`. + 5. Replaces the paramset file using `content` (fully overwrites the file). +2. The `git_commit` job runs: + 1. Commits updated files into the Instance repository. + +**Results:** + +1. Paramset file is replaced at the resolved path. +2. File content matches `paramsets[].content`. +3. Changes are committed. + +--- + +### UC-EINV-PS-3: Delete paramSet file + +**Pre-requisites:** + +1. The target ParamSet file exists + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `paramsets[]`: + +- `action: delete` +- `place: env | cluster | site` +- `content.name: ` +- `content` is a valid Paramset file content + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `paramSets[]` item against the request schema: + - `action == delete` + - `place == { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`: + - `place=env` → `/environments///Inventory/parameters/.yml` + - `place=cluster` → `/environments//Inventory/parameters/.yml` + - `place=site` → `/environments/Inventory/parameters/.yml` + 5. Deletes the target paramset file if it exists. + - Directories are not removed. + +2. The `git_commit` job runs: + 1. Commits repository changes into the Instance repository. + +**Results:** + +1. The paramset file is removed at the resolved path (if it existed). +2. Parent directories remain unchanged. +3. Changes are committed. + +--- + +## Environment Inventory: Credentials + +### UC-EINV-CR-1: Create credentials file (`create_or_replace`, file does not exist) + +**Pre-requisites:** + +1. The target credentials file does not exist (for the resolved `place`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `credentials[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content` is a credentials map (one or multiple credentials) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `credentials[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `content` is present + 3. Resolves target path by `place`: + - `place=env` → `/environments///Inventory/credentials/inventory_generation_creds.yml` + - `place=cluster` → `/environments//Inventory/credentials/inventory_generation_creds.yml` + - `place=site` → `/environments/credentials/inventory_generation_creds.yml` + 4. Creates `credentials/` directory if missing (for `env`/`cluster` levels). + 5. Creates the credentials file using `content` (create-or-replace semantics; in this UC the file is expected to be missing). +2. The `git_commit` job runs: + 1. Commits created files into the Instance repository. + +**Results:** + +1. Credentials file is created at the resolved path. +2. File content matches `credentials[].content`. +3. Changes are committed. + +--- + +### UC-EINV-CR-2: Replace credentials file (`create_or_replace`, file exists) + +**Pre-requisites:** + +1. The target credentials file exists (for the resolved `place`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `credentials[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content` is a credentials map (one or multiple credentials) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `credentials[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `content` is present + 3. Resolves target path by `place`. + 4. Replaces the credentials file using `content` (fully overwrites the file). +2. The `git_commit` job runs: + 1. Commits updated files into the Instance repository. + +**Results:** + +1. Credentials file is replaced at the resolved path. +2. File content matches `credentials[].content`. +3. Changes are committed. + +--- + +### UC-EINV-CR-3: Delete credentials file + +**Pre-requisites:** + +1. The target credentials file exists + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `credentials[]` with at least one item where: + +- `action: delete` +- `place: env | cluster | site` +- `content` is present + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `credentials[]` item against the request schema: + - `action == delete` + - `place ∈ { env, cluster, site }` + - `content` is present + 3. Resolves target credentials file path by `place`. + 4. Deletes the target credentials file if it exists. + - Directories are not removed. + +2. The `git_commit` job runs: + 1. Commits repository changes into the Instance repository. + +**Results:** + +1. Credentials file is removed at the resolved path (if it existed). +2. Parent directories remain unchanged (no directory cleanup). +3. Changes are committed. + +--- + +## Environment Inventory: Resource Profile Overrides + +### UC-EINV-RP-1: Create resource profile override file (`create_or_replace`, file does not exist) + +**Pre-requisites:** + +1. The target override file does not exist (for the resolved `place` and `content.name`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `resourceProfiles[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content.name: ` +- `content` is a valid Resource Profile Override file content + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `resourceProfiles[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`: + - `place=env` → `/environments///Inventory/resource_profiles/.yml` + - `place=cluster` → `/environments//Inventory/resource_profiles/.yml` + - `place=site` → `/environments/Inventory/resource_profiles/.yml` + 5. Creates `resource_profiles/` directory if missing. + 6. Creates the override file using `content` (create-or-replace semantics; in this UC the file is expected to be missing). +2. The `git_commit` job runs: + 1. Commits created files into the Instance repository. + +**Results:** + +1. Override file is created at the resolved path. +2. File content matches `resourceProfiles[].content`. +3. Changes are committed. + +--- + +### UC-EINV-RP-2: Replace resource profile override file (`create_or_replace`, file exists) + +**Pre-requisites:** + +1. The target override file exists (for the resolved `place` and `content.name`). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `resourceProfiles[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `content.name: ` +- `content` is a valid Resource Profile Override file content + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `resourceProfiles[]` item against the request schema: + - [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`. + 5. Replaces the override file using `content` (fully overwrites the file). +2. The `git_commit` job runs: + 1. Commits updated files into the Instance repository. + +**Results:** + +1. Override file is replaced at the resolved path. +2. File content matches `resourceProfiles[].content`. +3. Changes are committed. + +--- + +### UC-EINV-RP-3: Delete resource profile override file + +**Pre-requisites:** + +1. The target Resource Profile Override file exists + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `resourceProfiles[]` with at least one item where: + +- `action: delete` +- `place: env | cluster | site` +- `content.name: ` +- `content` is present (used to resolve the target filename) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `resourceProfiles[]` item against the request schema: + - `action == delete` + - `place ∈ { env, cluster, site }` + - `content` is present and contains `name` + 3. Extracts `` from `content.name`. + 4. Resolves target path by `place`: + - `place=env` → `/environments///Inventory/resource_profiles/.yml` + - `place=cluster` → `/environments//resource_profiles/.yml` + - `place=site` → `/environments/resource_profiles/.yml` + 5. Deletes the target override file if it exists. + - Directories are not removed. + +2. The `git_commit` job runs: + 1. Commits repository changes into the Instance repository. + +**Results:** + +1. Resource Profile Override file is removed at the resolved path (if it existed). +2. Parent directories remain unchanged (no directory cleanup). +3. Changes are committed. + +--- + +## Environment Inventory: Shared Template Variable Files + +### UC-EINV-STV-1: Create Shared Template Variable file (`create_or_replace`, file does not exist) + +**Pre-requisites:** + +1. The target Shared Template Variable file does not exist (for the resolved `place` and `name`): + - `place=env` → `/environments///shared-template-variables/.yml` + - `place=cluster` → `/environments//shared-template-variables/.yml` + - `place=site` → `/environments/shared-template-variables/.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: [`/docs/features/env-inventory-generation.md`](/docs/features/env-inventory-generation.md) + +`ENV_INVENTORY_CONTENT` includes `sharedTemplateVariables[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `name: ` +- `content` + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `sharedTemplateVariables[]` item against the request schema: + - `/docs/features/env-inventory-generation.md` + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `name` is present + - `content` is present + 3. Resolves target path by `place`: + - `place=env` → `/environments///shared-template-variables/.yml` + - `place=cluster` → `/environments//shared-template-variables/.yml` + - `place=site` → `/environments/shared-template-variables/.yml` + 4. Creates `shared-template-variables/` directory if missing. + 5. Creates the Shared Template Variable file using `content` (create-or-replace semantics; in this UC the file is expected to be missing). + +2. The `git_commit` job runs: + 1. Commits created files into the Instance repository. + +**Results:** + +1. Shared Template Variable file is created at the resolved path. +2. Filename is saved as `.yml`. +3. File content matches `sharedTemplateVariables[].content`. +4. Changes are committed. + +--- + +### UC-EINV-STV-2: Replace Shared Template Variable file (create_or_replace, file exists) + +**Pre-requisites:** + +1. The target Shared Template Variable file exists (for the resolved `place` and `name`): + - `place=env` → `/environments///shared-template-variables/.yml` + - `place=cluster` → `/environments//shared-template-variables/.yml` + - `place=site` → `/environments/shared-template-variables/.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: `/docs/features/env-inventory-generation.md` + +`ENV_INVENTORY_CONTENT` includes `sharedTemplateVariables[]` with at least one item where: + +- `action: create_or_replace` +- `place: env | cluster | site` +- `name: ` +- `content` + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `sharedTemplateVariables[]` item against the request schema: + - `/docs/features/env-inventory-generation.md` + - `action == create_or_replace` + - `place ∈ { env, cluster, site }` + - `name` is present + - `content` is present + 3. Resolves target path by `place`. + 4. Replaces the Shared Template Variable file using `content` (fully overwrites the file). + +2. The `git_commit` job runs: + 1. Commits updated files into the Instance repository. + +**Results:** + +1. Shared Template Variable file is replaced at the resolved path. +2. File content matches `sharedTemplateVariables[].content`. +3. Changes are committed. + +--- + +### UC-EINV-STV-3: Delete Shared Template Variable file + +**Pre-requisites:** + +1. The target Shared Template Variable file exists (for the resolved `place` and `name`): + - `place=env` → `/environments///shared-template-variables/.yml` + - `place=cluster` → `/environments//shared-template-variables/.yml` + - `place=site` → `/environments/shared-template-variables/.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: `/docs/features/env-inventory-generation.md` + +`ENV_INVENTORY_CONTENT` includes `sharedTemplateVariables[]` with at least one item where: + +- `action: delete` +- `place: env | cluster | site` +- `name: ` + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Validates the `sharedTemplateVariables[]` item against the request schema: + - `action == delete` + - `place ∈ { env, cluster, site }` + - `name` is present + 3. Resolves target path by `place`: + - `place=env` → `/environments///shared-template-variables/.yml` + - `place=cluster` → `/environments//shared-template-variables/.yml` + - `place=site` → `/environments/shared-template-variables/.yml` + 4. Deletes the target Shared Template Variable file if it exists. + - Directories are not removed. + +2. The `git_commit` job runs: + 1. Commits repository changes into the Instance repository. + +**Results:** + +1. Shared Template Variable file is removed at the resolved path (if it existed). +2. Parent directories remain unchanged (no directory cleanup). +3. Changes are committed. + +--- + +### UC-EINV-AT-ALL-1: Rollback all Inventory changes if any operation fails (negative, atomic processing) + +**Pre-requisites:** + +1. Instance pipeline is started with `ENV_INVENTORY_CONTENT` that includes one or more Inventory operations (any combination of): + - `envDefinition` + - `paramSets[]` + - `credentials[]` + - `resourceProfiles[]` + - `sharedTemplateVariables[]` +2. Repository has an initial state (files may exist or not exist). +3. At least one requested operation will fail during processing (validation error or file operation error). + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_INVENTORY_CONTENT: ` + - Examples: `/docs/features/env-inventory-generation.md` + +During processing of `ENV_INVENTORY_CONTENT`, at least one operation fails . + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads and parses `ENV_INVENTORY_CONTENT`. + 2. Runs validations: + - Parameter exclusivity validation: + - If `ENV_INVENTORY_CONTENT` is provided together with `ENV_INVENTORY_INIT` or `ENV_SPECIFIC_PARAMS`, validation fails. + - JSON schema validation: + - `ENV_INVENTORY_CONTENT` is validated against `/schemas/env-inventory-content.schema.json`. + 3. Starts atomic processing of all requested operations (order between object types is not guaranteed). + 4. Applies some operations (examples of partial progress): + - Creates required directories (e.g., `Inventory`, `parameters`, `credentials`, `resource_profiles`, `shared-template-variables`). + - Creates or replaces files (e.g., `env_definition.yml`, paramset files, credential files, resource profile overrides, shared template variables). + 5. While processing one of operations, an error occurs: + - Schema validation fails for one object content, **or** + - Any file write/delete operation fails. + 6. Performs rollback: + - Reverts all files created/changed during this job run. + - Restores overwritten files to their previous state. + - Removes directories/files created only during this run. + 7. Fails the job with a readable error message in logs. + +2. The `git_commit` job does **not** commit any changes (because there must be no net changes after rollback). + +**Results:** + +1. No files are modified in the Instance repository after the pipeline run ). +2. Any files created during this run are removed. +3. Any overwritten files are restored to the original state. +4. Any directories created only during this run are removed. +5. Pipeline logs contain a readable error message explaining the failure reason. +6. No changes are committed. + +--- + +## Template Version Update + +### UC-EINV-TV-1: Apply `ENV_TEMPLATE_VERSION` (`PERSISTENT` vs `TEMPORARY`) + +**Pre-requisites:** + +1. Environment Inventory exists: + - `/environments///Inventory/env_definition.yml` + +**Trigger:** + +Instance pipeline (GitLab or GitHub) is started with: + +- `ENV_NAMES: /` +- `ENV_TEMPLATE_VERSION: ` +- `ENV_TEMPLATE_VERSION_UPDATE_MODE: PERSISTENT | TEMPORARY` (optional; default: `PERSISTENT`) + +**Steps:** + +1. The `env_inventory_generation` job runs: + 1. Reads `ENV_TEMPLATE_VERSION_UPDATE_MODE` (default: `PERSISTENT`). + 2. Applies `ENV_TEMPLATE_VERSION`: + - **PERSISTENT**: + - Updates template version in `env_definition.yml` + (`envTemplate.artifact` or `envTemplate.templateArtifact.artifact.version`). + - **TEMPORARY**: + - Does not change `envTemplate.*` in `env_definition.yml`. + - Writes the applied version into: + - `generatedVersions.generateEnvironmentLatestVersion: ""` +2. The `git_commit` job runs: + 1. Commits updated `env_definition.yml` into the Instance repository. + +**Results:** + +1. **PERSISTENT**: template version in `env_definition.yml` is updated and committed. +2. **TEMPORARY**: `generatedVersions.generateEnvironmentLatestVersion` is updated and committed; `envTemplate.*` remains unchanged. diff --git a/github_workflows/instance-repo-pipeline/.github/configuration/config.env b/github_workflows/instance-repo-pipeline/.github/configuration/config.env index 640b0d3b1..8bc9db08b 100644 --- a/github_workflows/instance-repo-pipeline/.github/configuration/config.env +++ b/github_workflows/instance-repo-pipeline/.github/configuration/config.env @@ -1,4 +1,5 @@ CI_PROJECT_DIR=/workspace +INSTANCES_DIR=/workspace/environments COMMIT_ENV=true GIT_STRATEGY=none GITHUB_USER_EMAIL=envgene@qubership.org diff --git a/github_workflows/instance-repo-pipeline/.github/jobs/build_env_job.sh b/github_workflows/instance-repo-pipeline/.github/jobs/build_env_job.sh deleted file mode 100755 index 8f2c00e78..000000000 --- a/github_workflows/instance-repo-pipeline/.github/jobs/build_env_job.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -e - -# build_env_job.sh -# This script implements the logic from prepare_env_build_job function -# It performs environment build operations including certificate updates, -# template version setting, environment building, and cleanup - -# Activate Python virtual environment -source /module/venv/bin/activate - -# Enable debug mode if envgen_debug is set -if [ "${envgen_debug}" == "true" ]; then - set -o xtrace -fi - -# Update CA certificates if certs directory exists -if [ -d "${CI_PROJECT_DIR}/configuration/certs" ]; then - cert_path=$(ls -A "${CI_PROJECT_DIR}/configuration/certs") - for path in $cert_path; do - bash /module/scripts/update_ca_cert.sh "${CI_PROJECT_DIR}/configuration/certs/$path" - done -fi - -# Set template version if ENV_TEMPLATE_VERSION is provided and not in test mode -if [ -n "${ENV_TEMPLATE_VERSION}" ] && [ "${ENV_TEMPLATE_VERSION}" != "" ] && [ "${ENV_TEMPLATE_TEST}" != "true" ]; then - echo "Setting template version to ${ENV_TEMPLATE_VERSION}" - python3 /build_env/scripts/build_env/env_template/set_template_version.py -fi - -# Execute main.py to build the environment -echo "Executing main.py to build environment..." -cd /build_env -if ! python3 /build_env/scripts/build_env/main.py; then - echo "Failed to execute main.py" - exit 1 -fi - -# Determine environment name based on test mode -if [ "${ENV_TEMPLATE_TEST}" == "true" ]; then - if [ -f "set_variable.txt" ]; then - env_name=$(cat set_variable.txt) - # Replace envgeneNullValue with test_value in credentials - if [ -f "${CI_PROJECT_DIR}/environments/${env_name}/Credentials/credentials.yml" ]; then - sed -i "s|\"envgeneNullValue\"|\"test_value\"|g" "${CI_PROJECT_DIR}/environments/${env_name}/Credentials/credentials.yml" - fi - else - echo "Warning: set_variable.txt not found in template test mode" - exit 1 - fi -else - # Extract environment name from ENV_NAME (last part after /) - # shellcheck disable=SC2153 - env_name=$(echo "${ENV_NAME}" | awk -F '/' '{print $NF}') - export env_name -fi - -# Set permissions on Credentials files -# Use find without sudo if running as root, otherwise use sudo -env_path=$(find "${CI_PROJECT_DIR}/environments" -type d -name "${env_name}" 2>/dev/null || sudo find "${CI_PROJECT_DIR}/environments" -type d -name "${env_name}") - -for path in $env_path; do - if [ -d "${path}/Credentials" ]; then - # Try without sudo first, fallback to sudo if needed - chmod ugo+rw "${path}/Credentials"/* 2>/dev/null || sudo chmod ugo+rw "${path}/Credentials"/* || true - fi -done - -# After script: Copy tmp files from /build_env/tmp to CI_PROJECT_DIR/tmp -echo "Copying tmp files..." -mkdir -p "${CI_PROJECT_DIR}/tmp" -if [ -d "/build_env/tmp" ]; then - cp -r /build_env/tmp/* "${CI_PROJECT_DIR}/tmp" 2>/dev/null || true -fi - -echo "Build environment job completed successfully" diff --git a/github_workflows/instance-repo-pipeline/.github/pipeline_vars.env b/github_workflows/instance-repo-pipeline/.github/pipeline_vars.env index dc9be01a6..e69de29bb 100644 --- a/github_workflows/instance-repo-pipeline/.github/pipeline_vars.env +++ b/github_workflows/instance-repo-pipeline/.github/pipeline_vars.env @@ -1,11 +0,0 @@ -# ============================================================================== -# Pipeline Variables File -# ============================================================================== -# -# This file is used to set environment variables for the EnvGene pipeline. -# Variables defined here are loaded at the "Load environment variables -# from configuration files" step in the workflow and used in all subsequent steps. -# -# Example: -# -# BG_MANAGE=true diff --git a/github_workflows/instance-repo-pipeline/.github/scripts/create_env_generation_params.sh b/github_workflows/instance-repo-pipeline/.github/scripts/create_env_generation_params.sh index 166f0c10c..1866ef0c0 100755 --- a/github_workflows/instance-repo-pipeline/.github/scripts/create_env_generation_params.sh +++ b/github_workflows/instance-repo-pipeline/.github/scripts/create_env_generation_params.sh @@ -4,7 +4,7 @@ possible_vars=( "SD_VERSION" "SD_DATA" "SD_DELTA" - "ENV_SPECIFIC_PARAMETERS" + "ENV_SPECIFIC_PARAMS" "ENV_TEMPLATE_NAME" ) diff --git a/github_workflows/instance-repo-pipeline/.github/scripts/generate_env_matrix.sh b/github_workflows/instance-repo-pipeline/.github/scripts/generate_env_matrix.sh index 4684c5d46..db856b233 100755 --- a/github_workflows/instance-repo-pipeline/.github/scripts/generate_env_matrix.sh +++ b/github_workflows/instance-repo-pipeline/.github/scripts/generate_env_matrix.sh @@ -5,7 +5,10 @@ if [ -z "$ENV_NAMES" ]; then exit 1 fi -env_matrix=$(echo "$ENV_NAMES" | jq -R -s -c 'split(",") | map(select(. != "" and . != null)) | map(gsub("^\\s+|\\s+$"; "")) | map(select(. != ""))') +# Replace literal \n with comma +ENV_NAMES="${ENV_NAMES//\\n/,}" + +env_matrix=$(echo "$ENV_NAMES" | jq -R -s -c 'split(",") | map(split(";")) | flatten | map(split(" ")) | flatten | map(select(. != "" and . != null)) | map(gsub("^\\s+|\\s+$"; "")) | map(select(. != ""))') echo "env_matrix=$env_matrix" >>$GITHUB_OUTPUT echo "Generated matrix: $env_matrix" diff --git a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml index 3e27103b7..a85c10c27 100644 --- a/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml +++ b/github_workflows/instance-repo-pipeline/.github/workflows/Envgene.yml @@ -69,9 +69,9 @@ env: DOCKER_IMAGE_NAME_EFFECTIVE_SET_GENERATOR: "${{ vars.DOCKER_REGISTRY || 'ghcr.io/netcracker' }}/qubership-effective-set-generator" #DOCKER_IMAGE_TAGS - DOCKER_IMAGE_TAG_PIPEGENE: "1.18.0" - DOCKER_IMAGE_TAG_ENVGENE: "1.18.0" - DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.18.0" + DOCKER_IMAGE_TAG_PIPEGENE: "1.21.1" + DOCKER_IMAGE_TAG_ENVGENE: "1.21.1" + DOCKER_IMAGE_TAG_EFFECTIVE_SET_GENERATOR: "1.21.1" jobs: process_environment_variables: @@ -82,7 +82,7 @@ jobs: env_matrix: ${{ steps.matrix-generator.outputs.env_matrix }} BG_MANAGE: ${{ env.BG_MANAGE }} ENV_TEMPLATE_TEST: ${{ env.ENV_TEMPLATE_TEST }} - ENV_SPECIFIC_PARAMETERS: ${{ env.ENV_SPECIFIC_PARAMETERS }} + ENV_SPECIFIC_PARAMS: ${{ env.ENV_SPECIFIC_PARAMS }} ENV_TEMPLATE_NAME: ${{ env.ENV_TEMPLATE_NAME }} CRED_ROTATION_PAYLOAD: ${{ env.CRED_ROTATION_PAYLOAD }} ENV_BUILDER: ${{ env.ENV_BUILDER }} @@ -184,7 +184,7 @@ jobs: run: env > .env.container ### BG MANAGE ### - - name: BG Manage + - name: BG_MANAGE if: needs.process_environment_variables.outputs.BG_MANAGE == 'true' env: FULL_ENV_NAME: ${{ matrix.environment }} @@ -194,41 +194,51 @@ jobs: bash -c " source /module/venv/bin/activate - echo "Executing BG Manage..." + echo 'Executing BG Manage...' if ! python /scripts/bg_manage/bg_manage.py; then - echo "BG Manage failed with exit code $?" + echo 'BG Manage failed with exit code $?' exit 1 fi " + + - name: BG_MANAGE - Upload BG_MANAGE Package + uses: actions/upload-artifact@v4 + if: needs.process_environment_variables.outputs.BG_MANAGE == 'true' + with: + name: bg_manage_${{ env.PACKAGE_NAME }} + path: environments/${{ matrix.environment }} + include-hidden-files: true ########################## - ### GENERATE INVENTORY ### - - name: Generate Inventory - if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMETERS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') + + ### ENV_INVENTORY_GENERATION ### + - name: ENV_INVENTORY_GENERATION + if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ bash -c " source /module/venv/bin/activate - echo \"Executing inventory generation...\" + echo 'Executing inventory generation...' if ! python3 /build_env/scripts/build_env/env_inventory_generation.py; then echo \"Inventory generation failed with exit code \$?\" exit 1 fi " - - name: Upload Generate Inventory Package + - name: ENV_INVENTORY_GENERATION - Upload ENV_INVENTORY_GENERATION Package uses: actions/upload-artifact@v4 - if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMETERS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') + if: needs.process_environment_variables.outputs.ENV_TEMPLATE_TEST == 'false' && (needs.process_environment_variables.outputs.ENV_SPECIFIC_PARAMS != '' || needs.process_environment_variables.outputs.ENV_TEMPLATE_NAME != '') with: - name: generate_inventory_${{ env.PACKAGE_NAME }} + name: env_inventory_generation_${{ env.PACKAGE_NAME }} path: environments/${{ matrix.environment }} include-hidden-files: true ########################## + ### CREDENTIAL ROTATION ### - - name: Credential Rotation + - name: CREDENTIAL_ROTATION if: needs.process_environment_variables.outputs.CRED_ROTATION_PAYLOAD != '' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ @@ -236,14 +246,14 @@ jobs: bash -c " source /module/venv/bin/activate - echo \"Executing credential rotation...\" + echo 'Executing credential rotation...' if ! python3 /module/creds_rotation_scripts/creds_rotation_handler.py; then - echo \"Credential rotation failed with exit code \$?\" + echo 'Credential rotation failed with exit code \$?' exit 1 fi " - - name: Upload Credential Rotation Artifact + - name: CREDENTIAL_ROTATION - Upload Credential Rotation Artifact uses: actions/upload-artifact@v4 if: needs.process_environment_variables.outputs.CRED_ROTATION_PAYLOAD != '' with: @@ -252,25 +262,69 @@ jobs: include-hidden-files: true ########################## - ### BUILD ENV ### - - name: Build Env + + ### APP_REG_DEF_PROCESS ### + - name: APP_REG_DEF_PROCESS if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ - sh -c "cp ${CI_PROJECT_DIR}/.github/jobs/build_env_job.sh /tmp/build_env_job.sh && chmod +x /tmp/build_env_job.sh && bash /tmp/build_env_job.sh" + bash -c " + source /module/venv/bin/activate + echo 'Executing APP_REG_DEF_PROCESS...' + python3 /module/scripts/utils/log_pipe_params.py + /module/scripts/handle_certs.sh + python3 /build_env/scripts/build_env/env_template/set_template_version.py + python3 /build_env/scripts/build_env/appregdef_render.py + " - - name: Upload Build Env Package + - name: APP_REG_DEF_PROCESS - Upload APP_REG_DEF_PROCESS Package uses: actions/upload-artifact@v4 if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' with: - name: build_env_${{ env.PACKAGE_NAME }} - path: environments/${{ matrix.environment }} + name: app_reg_def_process_${{ env.PACKAGE_NAME }} + path: | + environments/${{ matrix.environment }} + configuration + tmp + include-hidden-files: true + ########################## + + + ### ENV_BUILD ### + - name: ENV_BUILD + if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' + run: | + docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ + ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ + bash -c " + source /module/venv/bin/activate + echo 'Executing ENV_BUILD...' + python /module/scripts/utils/log_pipe_params.py + /module/scripts/handle_certs.sh + cd /build_env; python3 /build_env/scripts/build_env/main.py + + if [ \"\$ENV_TEMPLATE_TEST\" == \"true\" ]; then + env_name=\$(cat set_variable.txt) + sed -i \"s|\\\\\\\"envgeneNullValue\\\\\\\"|\\\\\\\"test_value\\\\\\\"|g\" \"\${CI_PROJECT_DIR}/environments/\$env_name/Credentials/credentials.yml\" + fi + " + + - name: ENV_BUILD - Upload Build Env Package + uses: actions/upload-artifact@v4 + if: needs.process_environment_variables.outputs.ENV_BUILDER == 'true' + with: + name: env_build_${{ env.PACKAGE_NAME }} + path: | + environments/${{ matrix.environment }} + configuration + tmp include-hidden-files: true ########################## + ### GENERATE EFFECTIVE SET ### - - name: Generate Effective Set + - name: GENERATE_EFFECTIVE_SET if: needs.process_environment_variables.outputs.GENERATE_EFFECTIVE_SET == 'true' run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ @@ -293,7 +347,7 @@ jobs: done " - - name: Upload Generate Effective Set Package + - name: GENERATE_EFFECTIVE_SET - Upload Generate Effective Set Package uses: actions/upload-artifact@v4 if: needs.process_environment_variables.outputs.GENERATE_EFFECTIVE_SET == 'true' with: @@ -304,7 +358,7 @@ jobs: ### GIT COMMIT ### - - name: Git Commit + - name: GIT_COMMIT run: | docker run --rm -v ${{ github.workspace }}:${{ env.CI_PROJECT_DIR }} -w ${{ env.CI_PROJECT_DIR }} --env-file .env.container --user root -e HOME=/root \ ${{ env.DOCKER_IMAGE_NAME_ENVGENE }}:${{ env.DOCKER_IMAGE_TAG_ENVGENE }} \ @@ -336,10 +390,10 @@ jobs: " env > .env.container - - name: Upload Git Commit Package + - name: GIT_COMMIT - Upload Git Commit Package uses: actions/upload-artifact@v4 with: name: git_commit_${{ env.PACKAGE_NAME }} path: environments/${{ matrix.environment }} include-hidden-files: true - ########################## \ No newline at end of file + ########################## diff --git a/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml b/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml index 650258676..71f9a94d4 100644 --- a/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml +++ b/gsf_packages/envgene_discovery_project/git-system-follower-package/package.yaml @@ -1,5 +1,5 @@ apiVersion: v1 type: gitlab-ci-pipeline name: envgene_discovery_project -version: 1.18.0 +version: 1.21.1 dependencies: [] diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml b/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml index fc94c6bbd..31d3b135e 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/package.yaml @@ -1,5 +1,5 @@ apiVersion: v1 type: gitlab-ci-pipeline name: envgene_instance_project -version: 1.18.0 +version: 1.21.1 dependencies: [] diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json index 730198598..57bc739a8 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/cookiecutter.json @@ -2,7 +2,7 @@ "gsf_repository_name": "envgene_instance_project", "docker_registry": "ghcr.io", "docker_namespace": "netcracker", - "envgene_version": "1.18.0", + "envgene_version": "1.21.1", "envgen_image": "qubership-envgene", "pipe_image": "qubership-pipegene", "cloud_deploytool_image": "env-generator-deploytool_build_deploytool", diff --git a/python/envgene/envgenehelper/business_helper.py b/python/envgene/envgenehelper/business_helper.py index 5902fed6f..b8bb0aa5b 100644 --- a/python/envgene/envgenehelper/business_helper.py +++ b/python/envgene/envgenehelper/business_helper.py @@ -1,21 +1,20 @@ -from dataclasses import dataclass, field import re +from dataclasses import dataclass, field from os import getenv from pathlib import Path from typing import overload from ruyaml import CommentedMap +from ruyaml.scalarstring import DoubleQuotedScalarString +from .collections_helper import dump_as_yaml_format from .collections_helper import merge_lists -from .yaml_helper import findYamls, openYaml, yaml, writeYamlToFile, store_value_to_yaml, \ - validate_yaml_by_scheme_or_fail -from .json_helper import findJsons from .file_helper import getAbsPath, extractNameFromFile, check_file_exists, check_dir_exists, getParentDirName, \ extractNameFromDir -from .collections_helper import dump_as_yaml_format +from .json_helper import findJsons from .logger import logger -from ruyaml.scalarstring import DoubleQuotedScalarString -from pathlib import Path +from .yaml_helper import findYamls, openYaml, yaml, writeYamlToFile, store_value_to_yaml, \ + validate_yaml_by_scheme_or_fail # const INVENTORY_DIR_NAME = "Inventory" @@ -45,16 +44,21 @@ def find_env_instances_dir(env_name, instances_dir): logger.error(f"Directory for {env_name} is not found in {instances_dir}") raise ReferenceError(f"Can't find directory for {env_name}") + @overload def getenv_and_log(name: str) -> None | str: ... + + @overload def getenv_and_log(name: str, default: str) -> str: ... + def getenv_and_log(name, *args, **kwargs): var = getenv(name, *args, **kwargs) logger.info(f"{name}: {var}") return var + def getenv_with_error(var_name): var = getenv(var_name) if not var: @@ -62,9 +66,11 @@ def getenv_with_error(var_name): logger.debug(f"{var_name}: {var}") return var + def get_env_instances_dir(environment_name, cluster_name, instances_dir): return f"{instances_dir}/{cluster_name}/{environment_name}" + def get_current_env_dir_from_env_vars() -> Path: instances_dir = getenv_with_error('CI_PROJECT_DIR') env_name = getenv_with_error('FULL_ENV_NAME') @@ -72,6 +78,7 @@ def get_current_env_dir_from_env_vars() -> Path: logger.debug(env_dir_path) return env_dir_path + def check_environment_is_valid_or_fail(environment_name, cluster_name, instances_dir, skip_env_definition_check=False, validate_env_definition_by_schema=False, schemas_dir=""): env_dir = get_env_instances_dir(environment_name, cluster_name, instances_dir) @@ -360,6 +367,7 @@ def find_cloud_name_from_passport(source_env_dir, all_instances_dir): else: return "" + @dataclass class NamespaceFile: path: Path @@ -370,12 +378,14 @@ def __post_init__(self): self.definition_path = self.path.joinpath('namespace.yml') self.name = openYaml(self.definition_path)['name'] + def get_namespaces_path(env_dir: Path | None = None) -> Path: env_dir = env_dir or get_current_env_dir_from_env_vars() namespaces_path = env_dir.joinpath('Namespaces') logger.debug(namespaces_path) return namespaces_path + def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: namespaces_path = get_namespaces_path(env_dir) if not check_dir_exists(str(namespaces_path)): @@ -385,15 +395,20 @@ def get_namespaces(env_dir: Path | None = None) -> list[NamespaceFile]: logger.debug(namespaces) return namespaces -def get_bgd_path() -> Path: - env_dir = get_current_env_dir_from_env_vars() + +def get_bgd_path(env_dir: Path | None = None) -> Path: + env_dir = env_dir or get_current_env_dir_from_env_vars() bgd_path = env_dir.joinpath('bg_domain.yml') logger.debug(bgd_path) return bgd_path -def get_bgd_object() -> CommentedMap: - bgd_path = get_bgd_path() + +def get_bgd_object(env_dir: Path | None = None) -> CommentedMap: + bgd_path = get_bgd_path(env_dir) bgd_object = openYaml(bgd_path, allow_default=True) logger.debug(bgd_object) return bgd_object + +def parse_env_names(full_env_names: str): + return full_env_names.split("\n") diff --git a/python/envgene/envgenehelper/file_helper.py b/python/envgene/envgenehelper/file_helper.py index e1c2ec937..32f7174a8 100644 --- a/python/envgene/envgenehelper/file_helper.py +++ b/python/envgene/envgenehelper/file_helper.py @@ -57,7 +57,7 @@ def delete_dir(path): try: shutil.rmtree(path) except: - logger.info(f'{path} directory does not exist') + logger.warning(f'{path} directory does not exist') def is_glob(path: str) -> bool: @@ -138,10 +138,12 @@ def openFileAsString(filePath): def deleteFile(filePath): os.remove(filePath) + def deleteFileIfExists(filePath): if check_file_exists(filePath): os.remove(filePath) + def writeToFile(filePath, contents): os.makedirs(os.path.dirname(filePath), exist_ok=True) with open(filePath, 'w+') as f: @@ -251,3 +253,8 @@ def cleanup_dir(path: str): dir_path.mkdir(parents=True, exist_ok=True) logger.info(f"Directory has been recreated: {path}") + + +def is_dir_empty(dir_path): + dir_path = Path(dir_path) + return dir_path.exists() and dir_path.is_dir() and not any(dir_path.iterdir()) diff --git a/python/envgene/envgenehelper/yaml_helper.py b/python/envgene/envgenehelper/yaml_helper.py index e2ff4af21..029dcef7e 100644 --- a/python/envgene/envgenehelper/yaml_helper.py +++ b/python/envgene/envgenehelper/yaml_helper.py @@ -1,18 +1,20 @@ +import copy import json import pathlib -import os +from io import StringIO +from typing import OrderedDict + import jschon import jschon_tools -import ruyaml import jsonschema -import copy -from io import StringIO +import ruyaml +from ruyaml import CommentedMap, CommentedSeq +from ruyaml.scalarstring import DoubleQuotedScalarString, LiteralScalarString + from .file_helper import * -from .logger import logger from .json_helper import openJson -from ruyaml.scalarstring import DoubleQuotedScalarString, LiteralScalarString -from ruyaml import CommentedMap, CommentedSeq -from typing import Callable, OrderedDict +from .logger import logger +from jsonschema import RefResolver def create_yaml_processor(is_safe=False) -> ruyaml.main.YAML: @@ -44,6 +46,7 @@ def openYaml(filePath, safe_load=False, default_yaml: Callable = get_empty_yaml, resultYaml = readYaml(f.read(), safe_load, context=f"File: {filePath}") return resultYaml + def readYaml(text, safe_load=False, context=None) -> CommentedMap: if text is None: resultYaml = None @@ -229,7 +232,7 @@ def merge_yaml_into_target(yaml_content, target_attribute_str, source_yaml, over merge_yaml_into_target(target_yaml[k], "", v, overwrite_existing_values, overwrite_existing_comments) elif isinstance(target_yaml[k], list) and isinstance(v, list): target_yaml[k].extend(v_el for v_el in v if v_el not in target_yaml[k] and ( - isinstance(v_el, primitiveTypes) or isinstance(v_el, list))) + isinstance(v_el, primitiveTypes) or isinstance(v_el, list))) src_dicts = {} for v_k, v_el in enumerate(v): if isinstance(v_el, dict): @@ -376,16 +379,25 @@ def yaml_from_string(yaml_str): return result -def validate_yaml_by_scheme_or_fail(yaml_file_path: str, schema_file_path: str) -> None: - yaml_content = openYaml(yaml_file_path) - schema_content = openJson(schema_file_path) - errors = validate_yaml_data_by_scheme(yaml_content, schema_content) +def validate_yaml_by_scheme_or_fail(yaml_file_path: str = None, schema_file_path: str = None, + input_yaml_content: dict = None, input_schema_content: dict = None, + schemas_dir=None): + yaml_content = openYaml(yaml_file_path) if yaml_file_path else input_yaml_content + schema_content = openJson(schema_file_path) if schema_file_path else input_schema_content + + if schemas_dir: + base_uri = Path(schemas_dir).absolute().as_uri() + "/" + resolver = RefResolver(base_uri=base_uri, referrer=schema_content) + errors = validate_yaml_data_by_scheme(yaml_content, schema_content, resolver=resolver) + else: + errors = validate_yaml_data_by_scheme(yaml_content, schema_content) if len(errors) > 0: - rel_path = getRelPath(yaml_file_path) - logger.error(f"Validation of {rel_path} file has failed") + if yaml_file_path: + rel_path = getRelPath(yaml_file_path) + logger.error(f"Validation of {rel_path} file has failed") for err in errors: log_jsonschema_validation_error(err) - raise ValueError(f"Validation failed") from None + raise ValueError("Validation failed") from None def validate_yaml_data_by_scheme(data, schema, cls=None, *args, **kwargs): diff --git a/schemas/env-definition.schema.json b/schemas/env-definition.schema.json index c23441bb1..7febd9a7f 100644 --- a/schemas/env-definition.schema.json +++ b/schemas/env-definition.schema.json @@ -143,7 +143,7 @@ "sharedTemplateVariables": { "type": "array", "title": "Shared Template Variables", - "description": "Array of file names in 'shared-template-variables' folders", + "description": "Array of file names in 'shared_template_variables' folders", "items": { "type": "string" }, diff --git a/schemas/env-inventory-content.schema.json b/schemas/env-inventory-content.schema.json new file mode 100644 index 000000000..7dd0ec668 --- /dev/null +++ b/schemas/env-inventory-content.schema.json @@ -0,0 +1,209 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Environment Inventory Content", + "description": "Schema for ENV_INVENTORY_CONTENT parameter that allows to create, replace, and delete Environment Inventory and related objects", + "additionalProperties": false, + "properties": { + "envDefinition": { + "type": "object", + "title": "Environment Definition Block", + "description": "Block that controls env_definition.yml", + "additionalProperties": false, + "properties": { + "action": { + "type": "string", + "enum": ["create_or_replace", "delete"], + "title": "Action", + "description": "Operation mode for env_definition.yml" + }, + "content": { + "$ref": "env-definition.schema.json", + "title": "Environment Definition Content", + "description": "Full content of env_definition.yml" + } + }, + "required": ["action"], + "if": { + "properties": { + "action": { + "const": "create_or_replace" + } + } + }, + "then": { + "required": ["content"] + } + }, + "paramSets": { + "type": "array", + "title": "Parameter Sets", + "description": "Block that controls Parameter Set file operations", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "action": { + "type": "string", + "enum": ["create_or_replace", "delete"], + "title": "Action", + "description": "Operation mode for the target Parameter Set file" + }, + "place": { + "type": "string", + "enum": ["site", "cluster", "env"], + "title": "Place", + "description": "Defines where the Parameter Set file is stored" + }, + "content": { + "$ref": "paramset.schema.json", + "title": "Parameter Set Content", + "description": "Parameter Set definition as file content" + } + }, + "required": ["action", "place"], + "if": { + "properties": { + "action": { + "const": "create_or_replace" + } + } + }, + "then": { + "required": ["content"] + } + } + }, + "credentials": { + "type": "array", + "title": "Shared Credentials", + "description": "Block that controls Shared Credentials operations", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "action": { + "type": "string", + "enum": ["create_or_replace", "delete"], + "title": "Action", + "description": "Operation mode for the Shared Credentials file" + }, + "place": { + "type": "string", + "enum": ["site", "cluster", "env"], + "title": "Place", + "description": "Defines where the Shared Credentials file is stored" + }, + "content": { + "$ref": "credential.schema.json", + "title": "Shared Credentials Content", + "description": "Shared Credential as file content" + }, + "name": { + "type": "string", + "title": "Name", + "description": "Name of the Shared Credentials File (without extension). The file will be saved as .yml" + } + }, + "required": ["action", "place"], + "if": { + "properties": { + "action": { + "const": "create_or_replace" + } + } + }, + "then": { + "required": ["content"] + } + } + }, + "resourceProfiles": { + "type": "array", + "title": "Resource Profile Overrides", + "description": "List of Resource Profile Override operations", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "action": { + "type": "string", + "enum": ["create_or_replace", "delete"], + "title": "Action", + "description": "Operation mode for the Resource Profile Override file" + }, + "place": { + "type": "string", + "enum": ["site", "cluster", "env"], + "title": "Place", + "description": "Defines where the Resource Profile Override file is stored" + }, + "content": { + "$ref": "resource-profile.schema.json", + "title": "Resource Profile Override Content", + "description": "Resource Profile Override as file content" + } + }, + "required": ["action", "place"], + "if": { + "properties": { + "action": { + "const": "create_or_replace" + } + } + }, + "then": { + "required": ["content"] + } + } + }, + "sharedTemplateVariables": { + "type": "array", + "title": "Shared Template Variable Files", + "description": "Block that controls Shared Template Variable File operations", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "action": { + "type": "string", + "enum": ["create_or_replace", "delete"], + "title": "Action", + "description": "Operation mode for the Shared Template Variable File" + }, + "place": { + "type": "string", + "enum": ["site", "cluster", "env"], + "title": "Place", + "description": "Defines where the Shared Template Variable File is stored" + }, + "name": { + "type": "string", + "title": "File Name", + "description": "Name of the Shared Template Variable File (without extension). The file will be saved as .yml", + "pattern": "^[^/\\\\\\.\\.]+$", + "examples": ["prod-template-variables"] + }, + "content": { + "type": "object", + "title": "Shared Template Variable File Content", + "description": "Shared Template Variable File content as key-value hashmap. Must NOT be located in a parameters directory", + "additionalProperties": true + } + }, + "required": ["action", "place", "name"], + "if": { + "properties": { + "action": { + "const": "create_or_replace" + } + } + }, + "then": { + "required": ["content"] + } + } + } + } +} + diff --git a/scripts/build_env/appregdef_render.py b/scripts/build_env/appregdef_render.py new file mode 100644 index 000000000..9cded545f --- /dev/null +++ b/scripts/build_env/appregdef_render.py @@ -0,0 +1,44 @@ +import shutil + +from env_template.process_env_template import process_env_template +from envgenehelper import * +from render_config_env import EnvGenerator + + +if __name__ == '__main__': + template_version = process_env_template() + + cluster_name = getenv_with_error("CLUSTER_NAME") + env_name = getenv_with_error("ENVIRONMENT_NAME") + base_dir = getenv_with_error('CI_PROJECT_DIR') + instances_dir = getenv_with_error("INSTANCES_DIR") + + output_dir = f"{base_dir}/environments" + render_dir = f"/tmp/render/{env_name}" + templates_dir = f"{base_dir}/tmp/templates" + + env_dir = get_env_instances_dir(env_name, cluster_name, instances_dir) + cloud_passport_file_path = find_cloud_passport_definition(env_dir, instances_dir) + + render_context_vars = { + "cluster_name": cluster_name, + "output_dir": output_dir, + "current_env_dir": render_dir, + "templates_dir": templates_dir, + "cloud_passport_file_path": cloud_passport_file_path, + "env_instances_dir": env_dir + } + + render_context = EnvGenerator() + render_context.process_app_reg_defs(env_name, render_context_vars) + + for dir_name in ["AppDefs", "RegDefs"]: + src = Path(render_dir) / dir_name + dst = Path(env_dir) / dir_name + + if dst.exists(): + shutil.rmtree(dst) + if src.exists(): + shutil.move(src, dst) + + update_generated_versions(env_dir, BUILD_ENV_TAG, template_version) \ No newline at end of file diff --git a/scripts/build_env/env_inventory_generation.py b/scripts/build_env/env_inventory_generation.py index df36ef669..7b5c100bf 100644 --- a/scripts/build_env/env_inventory_generation.py +++ b/scripts/build_env/env_inventory_generation.py @@ -1,43 +1,53 @@ -from os import path +from enum import Enum import envgenehelper as helper import envgenehelper.logger as logger from envgenehelper import * from envgenehelper.business_helper import INV_GEN_CREDS_PATH from envgenehelper.env_helper import Environment +from typing_extensions import deprecated from create_credentials import CRED_TYPE_SECRET -# const PARAMSETS_DIR_PATH = "Inventory/parameters/" CLUSTER_TOKEN_CRED_ID = "cloud-deploy-sa-token" +INVENTORY = "Inventory" +DEPRECATED_MESSAGE = "Deprecated inventory generation approach" +SCHEMAS_DIR = Path(__file__).resolve().parents[2] / "schemas" -# Get schema path from environment variable or use default path -SCHEMAS_DIR = getenv("JSON_SCHEMAS_DIR", path.join(path.dirname(path.dirname(path.dirname(__file__))), "schemas")) -PARAMSET_SCHEMA_PATH = path.join(SCHEMAS_DIR, "paramset.schema.json") -with open(PARAMSET_SCHEMA_PATH, 'r') as f: - PARAMSET_SCHEMA = json.load(f) +def generate_env_new_approach(): + env_name = getenv_with_error('ENV_NAME') + cluster = getenv_with_error('CLUSTER_NAME') + logger.info(f"Starting env inventory generation for env: {env_name} in cluster: {cluster}") + env_inventory_content = json.loads(getenv_with_error('ENV_INVENTORY_CONTENT')) + env_inv_content_schema_path = path.join(SCHEMAS_DIR, "env-inventory-content.schema.json") + validate_yaml_by_scheme_or_fail(input_yaml_content=env_inventory_content, + schema_file_path=env_inv_content_schema_path, + schemas_dir=SCHEMAS_DIR) + + handle_env_inv_content(env_inventory_content) + + +@deprecated(DEPRECATED_MESSAGE) def generate_env(): base_dir = getenv_and_log('CI_PROJECT_DIR') env_name = getenv_and_log('ENV_NAME') cluster = getenv_and_log('CLUSTER_NAME') - params_json = getenv_and_log("ENV_GENERATION_PARAMS") - params = json.loads(params_json) + env_inventory_init = getenv('ENV_INVENTORY_INIT') + env_specific_params = getenv('ENV_SPECIFIC_PARAMS') + env_template_name = getenv('ENV_TEMPLATE_NAME') - env_inventory_init = params['ENV_INVENTORY_INIT'] - env_specific_params = params['ENV_SPECIFIC_PARAMETERS'] - env_template_name = params['ENV_TEMPLATE_NAME'] - env_template_version = params['ENV_TEMPLATE_VERSION'] + env_template_version = getenv('ENV_TEMPLATE_VERSION') env = Environment(base_dir, cluster, env_name) logger.info(f"Starting env inventory generation for env: {env.name} in cluster: {env.cluster}") handle_env_inventory_init(env, env_inventory_init, env_template_version) - handle_env_specific_params(env, env_specific_params) + handle_env_specific_params(env, env_specific_params, SCHEMAS_DIR) helper.set_nested_yaml_attribute(env.inventory, 'envTemplate.name', env_template_name) helper.writeYamlToFile(env.inventory_path, env.inventory) @@ -48,6 +58,7 @@ def generate_env(): helper.encrypt_file(env.inv_gen_creds_path) +@deprecated(DEPRECATED_MESSAGE) def handle_env_inventory_init(env, env_inventory_init, env_template_version): if env_inventory_init != "true": logger.info("ENV_INVENTORY_INIT is not set to 'true'. Skipping env inventory initialization") @@ -62,7 +73,8 @@ def handle_env_inventory_init(env, env_inventory_init, env_template_version): helper.set_nested_yaml_attribute(env.inventory, 'envTemplate.envSpecificParamsets', helper.get_empty_yaml()) -def handle_env_specific_params(env, env_specific_params): +@deprecated(DEPRECATED_MESSAGE) +def handle_env_specific_params(env, env_specific_params, schemas_dir): if not env_specific_params or env_specific_params == "": logger.info("ENV_SPECIFIC_PARAMS are not set. Skipping env inventory update") return @@ -87,7 +99,7 @@ def handle_env_specific_params(env, env_specific_params): helper.merge_yaml_into_target(env.inventory, 'envTemplate.envSpecificParamsets', envSpecificParamsets) logger.info("ENV_SPECIFIC_PARAMS env details ", vars(env)) handle_credentials(env, creds) - create_paramset_files(env, paramsets) + create_paramset_files(env, paramsets, schemas_dir) helper.set_nested_yaml_attribute(env.inventory, 'inventory.tenantName', tenantName) helper.set_nested_yaml_attribute(env.inventory, 'inventory.tenantName', tenantName) @@ -95,20 +107,23 @@ def handle_env_specific_params(env, env_specific_params): logger.info(f"ENV_SPECIFIC_PARAMS env details : {vars(env)}") -def create_paramset_files(env, paramsets): +@deprecated(DEPRECATED_MESSAGE) +def create_paramset_files(env, paramsets, schemas_dir): if not paramsets: return + PARAMSET_SCHEMA_PATH = path.join(schemas_dir, "paramset.schema.json") ps_dir_path = path.join(env.env_path, PARAMSETS_DIR_PATH) helper.check_dir_exist_and_create(ps_dir_path) logger.info(f"Creating paramsets in {ps_dir_path}") for k, v in paramsets.items(): - jsonschema.validate(v, PARAMSET_SCHEMA) + jsonschema.validate(v, openYaml(PARAMSET_SCHEMA_PATH)) filename = k + ".yml" ps_path = path.join(ps_dir_path, filename) helper.writeYamlToFile(ps_path, v) # overwrites file logger.info(f"Created paramset {filename}") +@deprecated(DEPRECATED_MESSAGE) def handle_credentials(env, creds): if not creds: return @@ -123,6 +138,7 @@ def handle_credentials(env, creds): sharedMasterCredentialFiles) +@deprecated(DEPRECATED_MESSAGE) def handle_cluster_params(env, cluster_params): if not cluster_params: return @@ -133,5 +149,92 @@ def handle_cluster_params(env, cluster_params): helper.set_nested_yaml_attribute(env.creds, 'cloud-deploy-sa-token', cred, is_overwriting=False) +class Action(Enum): + CREATE_OR_REPLACE = "create_or_replace" + DELETE = "delete" + + +class Place(Enum): + ENV = "env" + CLUSTER = "cluster" + SITE = "site" + + +def resolve_path(env_dir: Path, place: Place, subdir: str, name: str, inventory: str = "") -> Path: + if place is Place.ENV: + base = env_dir / inventory + elif place is Place.CLUSTER: + base = env_dir.parent + elif place is Place.SITE: + base = env_dir.parent.parent + else: + raise ValueError(place) + return base / subdir / f"{name}.yml" + + +def handle_objects(env_dir, objects, subdir, inventory="", encrypt=False): + if not objects: + logger.info(f"No objects for {subdir}, skipping") + return + + for obj in objects: + place = Place(obj["place"]) + action = Action(obj["action"]) + content = obj["content"] + + name = content["name"] if content.get("name") else obj["name"] + obj_path = resolve_path(env_dir, place, subdir, name, inventory) + + logger.info(f"Processing {subdir}, action={action.value}, place={place.value}. Target path: {obj_path}") + if action is Action.CREATE_OR_REPLACE: + writeYamlToFile(obj_path, content) + if encrypt: + encrypt_file(obj_path) + beautifyYaml(obj_path) + + elif action is Action.DELETE: + logger.info(f"Deleting file: {obj_path}") + deleteFileIfExists(obj_path) + + +def handle_env_def(env_dir: Path, env_def: dict | None): + env_template_version = getenv('ENV_TEMPLATE_VERSION') + if not env_def: + logger.info("env_definition is not provided, skipping") + return + + action = Action(env_def["action"]) + env_def_path = env_dir / INVENTORY / "env_definition.yml" + logger.info(f"Processing env_definition, action={action.value}. Target path: {env_def_path}") + if action is Action.DELETE: + logger.info(f"Deleting environment directory: {env_dir}") + delete_dir(env_dir) + else: + content = env_def.get("content") + if env_template_version: + logger.info(f"Overriding envTemplate.artifact with ENV_TEMPLATE_VERSION={env_template_version}") + content["envTemplate"]["artifact"] = env_template_version + writeYamlToFile(env_def_path, content) + beautifyYaml(env_def_path) + logger.info("env_definition.yml successfully created/updated") + + +def handle_env_inv_content(env_inventory_content: dict): + env_dir = Path(get_current_env_dir_from_env_vars()) + + handle_env_def(env_dir, env_inventory_content.get("envDefinition")) + + handle_objects(env_dir, env_inventory_content.get("paramSets"), "parameters", INVENTORY) + handle_objects(env_dir, env_inventory_content.get("credentials"), "credentials", INVENTORY, encrypt=True) + handle_objects(env_dir, env_inventory_content.get("resourceProfiles"), "resource_profiles", INVENTORY) + handle_objects(env_dir, env_inventory_content.get("sharedTemplateVariables"), "shared_template_variables") + + if __name__ == "__main__": - generate_env() + + if getenv('ENV_INVENTORY_CONTENT'): + logger.info("Using new inventory generation approach") + generate_env_new_approach() + else: + logger.info("Using old inventory generation approach") + generate_env() diff --git a/scripts/build_env/env_template/process_env_template.py b/scripts/build_env/env_template/process_env_template.py index 256092526..d4c91ff80 100644 --- a/scripts/build_env/env_template/process_env_template.py +++ b/scripts/build_env/env_template/process_env_template.py @@ -8,16 +8,16 @@ from env_template.template_testing import run_env_test_setup from envgenehelper import getEnvDefinition, fetch_cred_value, getAppDefinitionPath from envgenehelper import openYaml, getenv_with_error, logger -from envgenehelper import unpack_archive, get_cred_config - +from envgenehelper import unpack_archive, get_cred_config, check_dir_exist_and_create from render_config_env import render_obj_by_context, Context -artifact_dest = f"{tempfile.gettempdir()}/artifact.zip" -build_env_path = "/build_env" +ARTIFACT_DEST = f"{tempfile.gettempdir()}/artifact.zip" def parse_artifact_appver(env_definition: dict) -> [str, str]: - artifact_appver = env_definition['envTemplate'].get('artifact', '') + artifact_appver = env_definition.get('envTemplate', {}).get('artifact') + if not artifact_appver: + raise ValueError(f"Environment template artifact is empty or missing from env_definition: {env_definition}") logger.info(f"Environment template artifact version: {artifact_appver}") return artifact_appver.split(':') @@ -48,8 +48,16 @@ def extract_snapshot_version(url: str, snapshot_version: str) -> str: return name[pos:] -# logic downloading template by artifact definition -def download_artifact_new_logic(env_definition: dict) -> str: +def validate_url(url, group_id, artifact_id, version): + if not url: + raise ValueError( + f"artifact not found group_id={group_id}, " + f"artifact_id={artifact_id}, version={version}" + ) + + +# logic resolving template by artifact definition +def resolve_artifact_new_logic(env_definition: dict, template_dest: str) -> str: app_name, app_version = parse_artifact_appver(env_definition) base_dir = getenv_with_error('CI_PROJECT_DIR') @@ -77,20 +85,20 @@ def download_artifact_new_logic(env_definition: dict) -> str: raise ValueError(f"Invalid maven coordinates from deployment descriptor {dd_url}") repo_url = dd_config.get("configurations", [{}])[0].get("maven_repository") or dd_repo - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP) + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, cred) + validate_url(template_url, group_id, artifact_id, version) else: logger.info("Loading environment template artifact from zip directly...") group_id, artifact_id, version = app_def.group_id, app_def.artifact_id, app_version artifact_info = asyncio.run(artifact.check_artifact_async(app_def, FileExtension.ZIP, app_version, cred)) if artifact_info: template_url, _ = artifact_info + validate_url(template_url, group_id, artifact_id, version) if "-SNAPSHOT" in app_version: resolved_version = extract_snapshot_version(template_url, app_version) - if not template_url: - raise ValueError(f"artifact not found group_id={group_id}, artifact_id={artifact_id}, version={version}") logger.info(f"Environment template url has been resolved: {template_url}") - artifact.download(template_url, artifact_dest, cred) - unpack_archive(artifact_dest, build_env_path) + artifact.download(template_url, ARTIFACT_DEST, cred) + unpack_archive(ARTIFACT_DEST, template_dest) return resolved_version @@ -103,8 +111,8 @@ def render_creds() -> dict: return rendered -# logic downloading template by exact coordinates and repo, deprecated -def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str: +# logic resolving template by exact coordinates and repo, deprecated +def resolve_artifact_old_logic(env_definition: dict, template_dest: str) -> str: template_artifact = env_definition['envTemplate']['templateArtifact'] artifact_info = template_artifact['artifact'] @@ -115,7 +123,8 @@ def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str: repo_type = template_artifact['templateRepository'] registry_name = template_artifact['registry'] - registry_dict = openYaml(Path(f"{project_dir}/configuration/registry.yml")) # another registry model + registry_dict = openYaml( + Path(f"{getenv_with_error('CI_PROJECT_DIR')}/configuration/registry.yml")) # another registry model registry = registry_dict[registry_name] repo_url = registry.get(repo_type) dd_repo_url = registry.get(dd_repo_type) @@ -125,8 +134,9 @@ def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str: repository_password = fetch_cred_value(registry.get("password"), cred_config) cred = Credentials(username=repository_username, password=repository_password) + template_url = None resolved_version = dd_version - dd_url = artifact.check_artifact(dd_repo_url, group_id, artifact_id, dd_version, FileExtension.JSON) + dd_url = artifact.check_artifact(dd_repo_url, group_id, artifact_id, dd_version, FileExtension.JSON, cred) if dd_url: logger.info(f"Deployment descriptor url for environment template has been resolved: {dd_url}") if "-SNAPSHOT" in dd_version: @@ -138,17 +148,17 @@ def download_artifact_old_logic(env_definition: dict, project_dir: str) -> str: if not all([group_id, artifact_id, version]): raise ValueError(f"Invalid maven coordinates from deployment descriptor {dd_url}") - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP) + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, version, FileExtension.ZIP, cred) + validate_url(template_url, group_id, artifact_id, version) else: logger.info("Loading environment template artifact from zip directly...") - template_url = artifact.check_artifact(repo_url, group_id, artifact_id, dd_version, FileExtension.ZIP) + template_url = artifact.check_artifact(repo_url, group_id, artifact_id, dd_version, FileExtension.ZIP, cred) + validate_url(template_url, group_id, artifact_id, dd_version) if "-SNAPSHOT" in dd_version: resolved_version = extract_snapshot_version(template_url, dd_version) - if not template_url: - raise ValueError(f"artifact not found group_id={group_id}, artifact_id={artifact_id}, version={resolved_version}") logger.info(f"Environment template url has been resolved: {template_url}") - artifact.download(template_url, artifact_dest, cred) - unpack_archive(artifact_dest, build_env_path) + artifact.download(template_url, ARTIFACT_DEST, cred) + unpack_archive(ARTIFACT_DEST, template_dest) return resolved_version @@ -156,15 +166,18 @@ def process_env_template() -> str: env_template_test = os.getenv("ENV_TEMPLATE_TEST", "").lower() == "true" if env_template_test: run_env_test_setup() - project_dir = getenv_with_error("CI_PROJECT_DIR") + project_dir = getenv_with_error('CI_PROJECT_DIR') + template_dest = f"{project_dir}/tmp" cluster = getenv_with_error("CLUSTER_NAME") environment = getenv_with_error("ENVIRONMENT_NAME") env_dir = Path(f"{project_dir}/environments/{cluster}/{environment}") env_definition = getEnvDefinition(env_dir) + check_dir_exist_and_create(template_dest) + if 'artifact' in env_definition.get('envTemplate', {}): - logger.info("Use template downloading new logic") - return download_artifact_new_logic(env_definition) + logger.info("Use template resolving new logic") + return resolve_artifact_new_logic(env_definition, template_dest) else: - logger.info("Use template downloading old logic") - return download_artifact_old_logic(env_definition, project_dir) + logger.info("Use template resolving old logic") + return resolve_artifact_old_logic(env_definition, template_dest) diff --git a/scripts/build_env/env_template/template_testing.py b/scripts/build_env/env_template/template_testing.py index 6c0f440ac..14ca54666 100644 --- a/scripts/build_env/env_template/template_testing.py +++ b/scripts/build_env/env_template/template_testing.py @@ -18,11 +18,10 @@ def run_env_test_setup(): writeYamlToFile(configs_conf_path, yaml.safe_load("crypt: false\n")) env_template_art_vers = getenv_with_error("ENV_TEMPLATE_VERSION") - env_name = os.getenv("ENV_NAME") - project_dir = os.getenv("CI_PROJECT_NAME") + env_name = getenv_with_error("ENV_NAME") env_template_vers_split = env_template_art_vers.split(':')[1].replace('.', '_') cluster_example_url = os.getenv("ansible_var_clusterExampleUrl", "https://test-cluster.example.com") - tenant_name = f"template_testing_{project_dir}_{env_name}" + tenant_name = getenv_with_error("CLUSTER_NAME") definition_env_name = "env-test" env_template_version_normalized = f"{env_template_vers_split.replace('-', '_')}" @@ -62,11 +61,6 @@ def run_env_test_setup(): shutil.copy(env_definition_conf_path, version_dir_path / "env_definition.yml") env_name = f"{tenant_name}/{tenant_name}_{env_template_version_normalized}" - environment_name = f"{tenant_name}_{env_template_version_normalized}" - - for k, v in {"CLUSTER_NAME": tenant_name, "ENVIRONMENT_NAME": environment_name, "ENV_NAME": env_name}.items(): - os.environ[k] = v - logger.info("Env var set: %s=%s", k, v) set_variable_path = Path(f"{base_dir}/set_variable.txt") writeToFile(set_variable_path, env_name) diff --git a/scripts/build_env/main.py b/scripts/build_env/main.py index d91e64baa..8205d9441 100644 --- a/scripts/build_env/main.py +++ b/scripts/build_env/main.py @@ -4,13 +4,11 @@ from build_env import build_env, process_additional_template_parameters from cloud_passport import update_env_definition_with_cloud_name from create_credentials import create_credentials -from env_template.process_env_template import process_env_template from render_config_env import EnvGenerator from resource_profiles import get_env_specific_resource_profiles from filter_namespaces import apply_ns_build_filter -# const INVENTORY_DIR_NAME = "Inventory" ENV_DEFINITION_FILE_NAME = "env_definition.yml" PARAMSET_SCHEMA = "schemas/paramset.schema.json" @@ -97,12 +95,12 @@ def handle_template_override(render_dir): deleteFile(file) -def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all_instances_dir, output_dir, - g_template_version, work_dir): +def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all_instances_dir, output_dir, work_dir): # defining folders that will be used during generation - render_dir = getAbsPath('tmp/render') - render_parameters_dir = getAbsPath('tmp/parameters_templates') - render_profiles_dir = getAbsPath('tmp/resource_profiles') + base_dir = getenv_with_error('CI_PROJECT_DIR') + render_dir = f"{base_dir}/tmp/render" + render_parameters_dir = f"{base_dir}/tmp/parameters_templates" + render_profiles_dir = f"{base_dir}/tmp/resource_profiles" namespaces_path = get_namespaces_path() if check_dir_exists(str(namespaces_path.absolute())): @@ -166,7 +164,6 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all envvars["env_instances_dir"] = getAbsPath(render_env_dir) envvars["render_dir"] = getAbsPath(render_dir) envvars["render_parameters_dir"] = getAbsPath(render_parameters_dir) - envvars["template_version"] = g_template_version envvars["cloud_passport_file_path"] = find_cloud_passport_definition(source_env_dir, all_instances_dir) envvars["cmdb_url"] = cmdb_url envvars["output_dir"] = output_dir @@ -176,12 +173,10 @@ def build_environment(env_name, cluster_name, templates_dir, source_env_dir, all handle_template_override(render_dir) env_specific_resource_profile_map = get_env_specific_resource_profiles(source_env_dir, all_instances_dir, ENV_SPECIFIC_RESOURCE_PROFILE_SCHEMA) - # building env build_env(env_name, source_env_dir, render_parameters_dir, render_dir, render_profiles_dir, env_specific_resource_profile_map, all_instances_dir, render_context) resulting_dir = post_process_env_after_rendering(env_name, render_env_dir, source_env_dir, all_instances_dir, output_dir) - validate_appregdefs(render_dir, env_name) return resulting_dir @@ -268,64 +263,36 @@ def validate_parameter_files(param_files): return errors -def validate_appregdefs(render_dir, env_name): - appdef_dir = f"{render_dir}/{env_name}/AppDefs" - regdef_dir = f"{render_dir}/{env_name}/RegDefs" - - if os.path.exists(appdef_dir): - appdef_files = findAllYamlsInDir(appdef_dir) - if not appdef_files: - logger.info(f"No AppDef YAMLs found in {appdef_dir}") - for file in appdef_files: - logger.info(f"AppDef file: {file}") - validate_yaml_by_scheme_or_fail(file, "schemas/appdef.schema.json") - - if os.path.exists(regdef_dir): - regdef_files = findAllYamlsInDir(regdef_dir) - if not regdef_files: - logger.info(f"No RegDef YAMLs found in {regdef_dir}") - for file in regdef_files: - logger.info(f"RegDef file: {file}") - validate_yaml_by_scheme_or_fail(file, "schemas/regdef.schema.json") - - -def render_environment(env_name, cluster_name, templates_dir, all_instances_dir, output_dir, g_template_version, - work_dir): +def render_environment(env_name, cluster_name, templates_dir, all_instances_dir, output_dir, work_dir): logger.info(f'env: {env_name}') logger.info(f'cluster_name: {cluster_name}') logger.info(f'templates_dir: {templates_dir}') logger.info(f'instances_dir: {all_instances_dir}') logger.info(f'output_dir: {output_dir}') - logger.info(f'template_version: {g_template_version}') logger.info(f'work_dir: {work_dir}') - # checking that directory is valid + check_environment_is_valid_or_fail(env_name, cluster_name, all_instances_dir, validate_env_definition_by_schema=True) # searching for env directory in instances validate_parameters(templates_dir, all_instances_dir, cluster_name, env_name) env_dir = get_env_instances_dir(env_name, cluster_name, all_instances_dir) logger.info(f"Environment {env_name} directory is {env_dir}") - # build env - resulting_env_dir = build_environment(env_name, cluster_name, templates_dir, env_dir, all_instances_dir, output_dir, - g_template_version, work_dir) - # create credentials + + resulting_env_dir = build_environment(env_name, cluster_name, templates_dir, env_dir, all_instances_dir, + output_dir, work_dir) create_credentials(resulting_env_dir, env_dir, all_instances_dir) - # update versions - update_generated_versions(resulting_env_dir, BUILD_ENV_TAG, g_template_version) apply_ns_build_filter() if __name__ == "__main__": base_dir = getenv_with_error('CI_PROJECT_DIR') - template_version = process_env_template() - cluster = getenv("CLUSTER_NAME") + cluster = getenv_with_error("CLUSTER_NAME") environment = getenv_with_error("ENVIRONMENT_NAME") - g_templates_dir = "/build_env/templates" + g_templates_dir = f"{base_dir}/tmp/templates" g_all_instances_dir = f"{base_dir}/environments" g_output_dir = f"{base_dir}/environments" g_work_dir = get_parent_dir_for_dir(g_all_instances_dir) - + decrypt_all_cred_files_for_env() - render_environment(environment, cluster, g_templates_dir, g_all_instances_dir, g_output_dir, - template_version, g_work_dir) + render_environment(environment, cluster, g_templates_dir, g_all_instances_dir, g_output_dir, g_work_dir) encrypt_all_cred_files_for_env() diff --git a/scripts/build_env/render_config_env.py b/scripts/build_env/render_config_env.py index 31b736893..6e8c02e79 100644 --- a/scripts/build_env/render_config_env.py +++ b/scripts/build_env/render_config_env.py @@ -1,16 +1,10 @@ -import os -import re -from collections import OrderedDict from collections.abc import Iterable from contextlib import contextmanager from datetime import datetime -from pathlib import Path from typing import Optional from deepmerge import always_merger -from envgenehelper import getEnvDefinition -from envgenehelper import logger, openYaml, readYaml, writeYamlToFile, openFileAsString, copy_path, dumpYamlToStr, \ - create_yaml_processor, find_all_yaml_files_by_stem, ensure_directory, dump_as_yaml_format +from envgenehelper import * from envgenehelper.business_helper import get_bgd_object, get_namespaces from envgenehelper.validation import ensure_valid_fields, ensure_required_keys from jinja2 import Template, TemplateError @@ -19,6 +13,8 @@ from jinja.jinja import create_jinja_env from jinja.replace_ansible_stuff import replace_ansible_stuff, escaping_quotation +SCHEMAS_DIR = Path(__file__).resolve().parents[2] / "schemas" + yml = create_yaml_processor() @@ -86,7 +82,8 @@ def render_obj_by_context(template: dict, context: Context) -> dict: class EnvGenerator: def __init__(self): self.ctx = Context() - logger.debug("EnvGenerator initialized with context: %s", self.ctx.dict(exclude_none=True, exclude={"env_vars"})) + logger.debug("EnvGenerator initialized with context: %s", + self.ctx.dict(exclude_none=True, exclude={"env_vars"})) def set_inventory(self): env_definition = getEnvDefinition(self.ctx.env_instances_dir) @@ -117,6 +114,18 @@ def set_env_template(self): logger.info(f"env_template = {env_template}") self.ctx.current_env_template = env_template + def setup_base_context(self, extra_env: dict): + all_vars = dict(os.environ) + self.ctx.update(extra_env) + self.ctx.env_vars.update(all_vars) + + self.set_inventory() + self.set_cloud_passport() + self.generate_config() + + current_env = self.ctx.config["environment"] + self.ctx.current_env = current_env + def validate_applications(self): applications = self.ctx.sd_config.get("applications", []) @@ -140,7 +149,7 @@ def validate_bgd(self): namespace_names = [ns.name for ns in get_namespaces()] bgd = get_bgd_object() mismatch = "" - for k,v in bgd.items(): + for k, v in bgd.items(): if 'Namespace' not in k: continue if v['name'] not in namespace_names: @@ -150,14 +159,33 @@ def validate_bgd(self): raise ValueError(f'Next namespaces were not found in available namespaces: {mismatch}') logger.info('Validation was successful') - def generate_ns_postfix(self, ns, ns_template_path) -> str: + def generate_ns_postfix(self, ns, ns_template_path, override_template_ns_name: str | None = None) -> str: deploy_postfix = ns.get("deploy_postfix") if deploy_postfix: ns_template_name = deploy_postfix else: # get base name(deploy postfix) without extensions ns_template_name = self.get_template_name(ns_template_path) - return ns_template_name + + ns_name = None + if override_template_ns_name: + ns_name = override_template_ns_name + elif ns_template_path: + rendered_ns = self.render_from_file_to_obj(ns_template_path) + ns_name = rendered_ns.get("name") + bgd = get_bgd_object(Path(f'{self.ctx.current_env_dir}')) + logger.debug(f'bgd object before comparing with ns: {bgd}') + if not bgd: + return ns_template_name + + origin_name = bgd["originNamespace"]["name"] + peer_name = bgd["peerNamespace"]["name"] + if ns_name == origin_name: + ns_template_name += "-origin" + elif ns_name == peer_name: + ns_template_name += "-peer" + logger.debug(f'After appending the ns name : {ns_template_name}') + return ns_template_name def generate_solution_structure(self): sd_path_stem = f'{self.ctx.current_env_dir}/Inventory/solution-descriptor/sd' @@ -260,20 +288,30 @@ def generate_bgd_file(self): return self.render_from_file_to_file(Template(template).render(self.ctx.as_dict()), target_path) + def fetch_template_override_name(self, ns) -> str: + override_namespace_content = ns.get("template_override") + if override_namespace_content: + rendered = create_jinja_env().from_string(str(override_namespace_content)).render(self.ctx.as_dict()) + if rendered: + template_name = readYaml(rendered) + return template_name.get("name") + return "" + def generate_namespace_file(self): context = self.ctx.as_dict() namespaces = self.ctx.current_env_template["namespaces"] for ns in namespaces: ns_template_path = Template(ns["template_path"]).render(context) - ns_template_name = self.generate_ns_postfix(ns, ns_template_path) - logger.info(f"Generate Namespace yaml for {ns_template_name}") + override_template_ns_name = self.fetch_template_override_name(ns) + deploy_postfx = self.generate_ns_postfix(ns, ns_template_path, override_template_ns_name) + logger.info(f"Generate Namespace yaml for {deploy_postfx}") current_env_dir = self.ctx.current_env_dir - ns_dir = f'{current_env_dir}/Namespaces/{ns_template_name}' + ns_dir = f'{current_env_dir}/Namespaces/{deploy_postfx}' namespace_file = f'{ns_dir}/namespace.yml' self.render_from_file_to_file(ns_template_path, namespace_file) self.generate_override_template(ns.get("template_override"), Path(f'{ns_dir}/namespace.yml_override'), - ns_template_name) + deploy_postfx) def calculate_cloud_name(self) -> str: inv = self.ctx.env_definition["inventory"] @@ -331,7 +369,7 @@ def generate_paramset_templates(self): def find_templates(self, path: str, patterns) -> list[Path]: path = Path(path) if not path.exists(): - logger.info(f"Templates directory not found: {path}") + logger.warning(f"Templates directory not found: {path}") return [] templates = [] for pattern in patterns: @@ -379,44 +417,49 @@ def set_appreg_def_overrides(self): output_dir = Path(self.ctx.output_dir) cluster_name = self.ctx.cluster_name config_file_name = Path("configuration") / "appregdef_config" - config_file_name_yaml = f"{config_file_name}.yaml" - config_file_name_yml = f"{config_file_name}.yml" - - potential_config_files = [ - output_dir / cluster_name / config_file_name_yaml, - output_dir / cluster_name / config_file_name_yml, - output_dir.parent / config_file_name_yaml, - output_dir.parent / config_file_name_yml, + + potential_global_config_files = [ + output_dir / f"{config_file_name}.yaml", + output_dir / f"{config_file_name}.yml", ] - appregdef_config_paths = [f for f in potential_config_files if f.exists()] - if appregdef_config_paths: - appregdef_config = {} - appregdef_config_path = appregdef_config_paths[0] + + potential_cluster_config_files = [ + output_dir / cluster_name / f"{config_file_name}.yaml", + output_dir / cluster_name / f"{config_file_name}.yml", + ] + + global_config_paths = [f for f in potential_global_config_files if f.exists()] + cluster_config_paths = [f for f in potential_cluster_config_files if f.exists()] + + if not global_config_paths and not cluster_config_paths: + logger.info("No appregdef_config file found, skipping overrides") + return + + global_appdefs, global_regdefs = {}, {} + cluster_appdefs, cluster_regdefs = {}, {} + + if len(global_config_paths) > 0: + global_path = global_config_paths[0] try: - appregdef_config = openYaml(appregdef_config_path) - logger.info(f"Overrides applications/registries definitions config found at: {appregdef_config_path}") + cfg = openYaml(global_path) + global_appdefs = cfg.get("appdefs", {}).get("overrides", {}) + global_regdefs = cfg.get("regdefs", {}).get("overrides", {}) + logger.info(f"Loaded global config from {global_path}") except Exception as e: - logger.warning(f"Failed to load config at: {appregdef_config_path}. Error: {e}") + logger.warning(f"Failed to load global config: {e}") - appdefs = self.ctx.appdefs - regdefs = self.ctx.regdefs - appdefs["overrides"] = appregdef_config.get(appdefs, {}).get("overrides", {}) - regdefs["overrides"] = appregdef_config.get(regdefs, {}).get("overrides", {}) + if len(cluster_config_paths) > 0: + cluster_path = cluster_config_paths[0] + try: + cfg = openYaml(cluster_path) + cluster_appdefs = cfg.get("appdefs", {}).get("overrides", {}) + cluster_regdefs = cfg.get("regdefs", {}).get("overrides", {}) + logger.info(f"Loaded cluster config from {cluster_path}") + except Exception as e: + logger.warning(f"Failed to load cluster config: {e}") - def process_app_reg_defs(self): - current_env_dir = self.ctx.current_env_dir - templates_dir = self.ctx.templates_dir - patterns = ["*.yaml.j2", "*.yml.j2", "*.j2", "*.yaml", "*.yml"] - appdef_templates = self.find_templates(f"{templates_dir}/appdefs", patterns) - regdef_templates = self.find_templates(f"{templates_dir}/regdefs", patterns) - self.ctx.appdef_templates = appdef_templates - self.ctx.regdef_templates = regdef_templates - - ensure_directory(Path(current_env_dir).joinpath("AppDefs"), 0o755) - ensure_directory(Path(current_env_dir).joinpath("RegDefs"), 0o755) - self.set_appreg_def_overrides() - self.render_app_defs() - self.render_reg_defs() + self.ctx.appdefs["overrides"] = always_merger.merge(global_appdefs, cluster_appdefs) + self.ctx.regdefs["overrides"] = always_merger.merge(global_regdefs, cluster_regdefs) def generate_profiles(self, profile_names: Iterable[str]): logger.info(f"Start rendering profiles from list: {profile_names}") @@ -427,23 +470,61 @@ def generate_profiles(self, profile_names: Iterable[str]): if template_name in profile_names: self.render_from_file_to_file(template_path, self.get_rendered_target_path(template_path)) + def validate_appregdefs(self): + render_dir = self.ctx.current_env_dir + + appdef_dir = f"{render_dir}/AppDefs" + regdef_dir = f"{render_dir}/RegDefs" + + if os.path.exists(appdef_dir): + appdef_files = findAllYamlsInDir(appdef_dir) + if not appdef_files: + logger.warning(f"No AppDef YAMLs found in {appdef_dir}") + for file in appdef_files: + logger.info(f"AppDef file: {file}") + validate_yaml_by_scheme_or_fail(file, str(SCHEMAS_DIR / "appdef.schema.json")) + + if os.path.exists(regdef_dir): + regdef_files = findAllYamlsInDir(regdef_dir) + if not regdef_files: + logger.warning(f"No RegDef YAMLs found in {regdef_dir}") + for file in regdef_files: + logger.info(f"RegDef file: {file}") + validate_yaml_by_scheme_or_fail(file, str(SCHEMAS_DIR / "regdef.schema.json")) + + def process_app_reg_defs(self, env_name: str, extra_env: dict): + logger.info( + f"Starting rendering app_reg_defs for {env_name}. Input params are:\n{dump_as_yaml_format(extra_env)}") + with self.ctx.use(): + self.setup_base_context(extra_env) + + current_env_dir = self.ctx.current_env_dir + templates_dir = self.ctx.templates_dir + patterns = ["*.yaml.j2", "*.yml.j2", "*.j2", "*.yaml", "*.yml"] + appdef_templates = self.find_templates(f"{templates_dir}/appdefs", patterns) + regdef_templates = self.find_templates(f"{templates_dir}/regdefs", patterns) + self.ctx.appdef_templates = appdef_templates + self.ctx.regdef_templates = regdef_templates + + ensure_directory(Path(current_env_dir).joinpath("AppDefs"), 0o755) + ensure_directory(Path(current_env_dir).joinpath("RegDefs"), 0o755) + self.set_appreg_def_overrides() + self.render_app_defs() + self.render_reg_defs() + + self.validate_appregdefs() + def render_config_env(self, env_name: str, extra_env: dict): logger.info(f"Starting rendering environment {env_name}. Input params are:\n{dump_as_yaml_format(extra_env)}") with self.ctx.use(): - all_vars = dict(os.environ) - self.ctx.update(extra_env) - self.ctx.env_vars.update(all_vars) - self.set_inventory() - self.set_cloud_passport() - self.generate_config() - - current_env = self.ctx.config["environment"] - self.ctx.current_env = current_env + self.setup_base_context(extra_env) + all_vars = self.ctx.env_vars + current_env = self.ctx.current_env self.ctx.cloud = self.calculate_cloud_name() self.ctx.tenant = current_env.get("tenant", '') self.ctx.deployer = current_env.get('deployer', '') - self.ctx.bgd = current_env.get('bg_domain','') + self.ctx.bgd = current_env.get('bg_domain', '') logger.info(f"current_env = {current_env}") self.ctx.update({ @@ -466,12 +547,12 @@ def render_config_env(self, env_name: str, extra_env: dict): self.ctx.current_env_dir = current_env_dir self.set_env_template() + self.generate_bgd_file() self.generate_solution_structure() self.generate_tenant_file() self.generate_cloud_file() self.generate_namespace_file() self.generate_composite_structure() - self.generate_bgd_file() env_specific_schema = self.ctx.current_env_template.get("envSpecificSchema") if env_specific_schema: @@ -480,7 +561,6 @@ def render_config_env(self, env_name: str, extra_env: dict): ensure_required_keys(self.ctx.as_dict(), required=["templates_dir", "env_instances_dir", "cluster_name", "current_env_dir"]) - self.process_app_reg_defs() logger.info(f"Rendering of templates for environment {env_name} generation was successful") self.validate_bgd() diff --git a/scripts/build_env/tests/app_reg_defs/test_appregdefs.py b/scripts/build_env/tests/app_reg_defs/test_appregdefs.py new file mode 100644 index 000000000..bd98f04e4 --- /dev/null +++ b/scripts/build_env/tests/app_reg_defs/test_appregdefs.py @@ -0,0 +1,142 @@ +import os +import shutil +from jinja2.exceptions import TemplateSyntaxError +from pathlib import Path +import pytest +import yaml +from render_config_env import EnvGenerator + + +class TestAppRegDefRendering: + + @pytest.fixture(scope="class", autouse=True) + def setup_test_environment(self, request): + cls = request.cls + + test_base = Path(__file__).parents[4] / "test_data" / "test_app_reg_defs" + cls.test_data_dir = test_base + cls.cluster_name = "cluster-01" + cls.env_name = "env-01" + + cls.output_dir = Path("/tmp/appregdefs") + cls.output_dir.mkdir(parents=True, exist_ok=True) + + os.environ["CLUSTER_NAME"] = cls.cluster_name + os.environ["ENVIRONMENT_NAME"] = cls.env_name + + yield + + os.environ.pop("CLUSTER_NAME", None) + os.environ.pop("ENVIRONMENT_NAME", None) + + def _get_test_case_dir(self, test_number: str) -> Path: + return self.test_data_dir / test_number + + def _setup_render_dir(self) -> Path: + render_dir = self.output_dir / "render" / self.env_name + if render_dir.exists(): + shutil.rmtree(render_dir) + render_dir.mkdir(parents=True, exist_ok=True) + return render_dir + + def _get_render_context(self, test_number: str) -> dict: + render_dir = self.output_dir / "render" / self.env_name + + test_case_dir = self._get_test_case_dir(test_number) + + env_dir = test_case_dir / "environments" / self.cluster_name / self.env_name + templates_dir = test_case_dir / "templates" + + return { + "cluster_name": self.cluster_name, + "output_dir": str(test_case_dir / "environments"), + "current_env_dir": str(render_dir), + "templates_dir": str(templates_dir), + "cloud_passport_file_path": "", + "env_instances_dir": str(env_dir) + } + + def _verify_rendered_files(self, test_number: str, render_dir: Path): + """Verify rendered files match expected results""" + test_case_dir = self._get_test_case_dir(test_number) + expected_appdefs = test_case_dir / "expected" / "appdefs" + expected_regdefs = test_case_dir / "expected" / "regdefs" + + if expected_appdefs.exists(): + for expected_file in expected_appdefs.glob("*.y*ml"): + base_name = expected_file.stem + rendered_file = None + for ext in ['.yml', '.yaml']: + candidate = render_dir / "AppDefs" / f"{base_name}{ext}" + if candidate.exists(): + rendered_file = candidate + break + + assert rendered_file and rendered_file.exists(), \ + f"AppDef file {expected_file.name} should be rendered (checked {base_name}.yml and {base_name}.yaml)" + + with open(expected_file, encoding="utf-8") as f: + expected_content = yaml.safe_load(f) + with open(rendered_file, encoding="utf-8") as f: + rendered_content = yaml.safe_load(f) + + assert rendered_content == expected_content, \ + f"AppDef {expected_file.name} content mismatch.\nExpected: {expected_content}\nGot: {rendered_content}" + + if expected_regdefs.exists(): + for expected_file in expected_regdefs.glob("*.y*ml"): + base_name = expected_file.stem + rendered_file = None + for ext in ['.yml', '.yaml']: + candidate = render_dir / "RegDefs" / f"{base_name}{ext}" + if candidate.exists(): + rendered_file = candidate + break + + assert rendered_file and rendered_file.exists(), \ + f"RegDef file {expected_file.name} should be rendered (checked {base_name}.yml and {base_name}.yaml)" + + with open(expected_file) as f: + expected_content = yaml.safe_load(f) + with open(rendered_file) as f: + rendered_content = yaml.safe_load(f) + + assert rendered_content == expected_content, \ + f"RegDef {expected_file.name} content mismatch.\nExpected: {expected_content}\nGot: {rendered_content}" + + POSITIVE_CASES = [ + "TC-001-001", + "TC-001-002", + "TC-001-003", + "TC-001-004", + "TC-001-005", + "TC-001-006", + "TC-001-008", + ] + + @pytest.mark.parametrize("test_number", POSITIVE_CASES) + def test_positive_basic_appdef_rendering(self, test_number): + self._setup_render_dir() + + render_context = EnvGenerator() + context_vars = self._get_render_context(test_number) + render_context.process_app_reg_defs(self.env_name, context_vars) + + render_dir = Path(context_vars["current_env_dir"]) + self._verify_rendered_files(test_number, render_dir) + + NEGATIVE_CASES = { + "TC-001-010": TemplateSyntaxError, + "TC-001-011": ValueError, + "TC-001-012": ValueError, + } + + @pytest.mark.parametrize("test_number,expected_exception", NEGATIVE_CASES.items()) + def test_negative_appregdef_rendering(self, test_number, expected_exception): + self._setup_render_dir() + + render_context = EnvGenerator() + context_vars = self._get_render_context(test_number) + with pytest.raises(expected_exception): + render_context.process_app_reg_defs(self.env_name, context_vars) + \ No newline at end of file diff --git a/scripts/build_env/tests/base_test.py b/scripts/build_env/tests/base_test.py new file mode 100644 index 000000000..e170370a5 --- /dev/null +++ b/scripts/build_env/tests/base_test.py @@ -0,0 +1,16 @@ +import os +from pathlib import Path + + +class BaseTest: + base_dir = Path(__file__).resolve().parents[3] + test_data_dir = base_dir / "test_data" + ci_project_dir = test_data_dir + expected_dir = test_data_dir + os.environ['CI_PROJECT_DIR'] = str(test_data_dir) + os.environ['JSON_SCHEMAS_DIR'] = str(base_dir / "schemas") + + def set_ci_project_dir(self, *subdirs) -> Path: + self.ci_project_dir = self.ci_project_dir.joinpath(*subdirs) + os.environ['CI_PROJECT_DIR'] = str(self.ci_project_dir) + return self.ci_project_dir diff --git a/scripts/build_env/tests/env-build/test_render_envs.py b/scripts/build_env/tests/env-build/test_render_envs.py index 251ef9945..a01f2ac6c 100644 --- a/scripts/build_env/tests/env-build/test_render_envs.py +++ b/scripts/build_env/tests/env-build/test_render_envs.py @@ -6,6 +6,8 @@ from main import render_environment, cleanup_resulting_dir from envgenehelper.test_helpers import TestHelpers +from tests.base_test import BaseTest + test_data = [ # (cluster_name, environment_name, template) ("cluster-01", "env-01", "composite-prod"), @@ -15,62 +17,28 @@ ("cluster01", "env01", "test-01"), ("cluster01", "env03", "test-template-1"), ("cluster01", "env04", "test-template-2"), - ("bgd-cluster","bgd-env","bgd"), + ("bgd-cluster", "bgd-env", "bgd"), ("cluster03", "rpo-replacement-mode", "simple"), ] -base_dir = Path(__file__).resolve().parents[4] -g_templates_dir = str((base_dir / "test_data/test_templates").resolve()) -g_inventory_dir = str((base_dir / "test_data/test_environments").resolve()) -g_output_dir = str((base_dir / "/tmp/test_environments").resolve()) -g_base_dir = get_parent_dir_for_dir(g_inventory_dir) -environ['CI_PROJECT_DIR'] = g_base_dir -os.environ['CI_COMMIT_REF_NAME'] = "branch_name" - - -@pytest.fixture(autouse=True) -def change_test_dir(monkeypatch): - monkeypatch.chdir(base_dir) - - -@pytest.mark.parametrize("cluster_name, env_name, version", test_data) -def test_render_envs(cluster_name, env_name, version): - environ['CI_PROJECT_DIR'] = g_base_dir - environ['FULL_ENV_NAME'] = cluster_name + '/' + env_name - render_environment(env_name, cluster_name, g_templates_dir, g_inventory_dir, g_output_dir, version, g_base_dir) - source_dir = f"{g_inventory_dir}/{cluster_name}/{env_name}" - generated_dir = f"{g_output_dir}/{cluster_name}/{env_name}" - files_to_compare = get_all_files_in_dir(source_dir) - logger.info(dump_as_yaml_format(files_to_compare)) - TestHelpers.assert_dirs_content(source_dir, generated_dir, True, False) - - -def setup_test_dir(tmp_path): - tmp_path.mkdir(exist_ok=True, parents=True) - dirs = ["Applications", "Namespaces", "Profiles"] - for d in dirs: - (tmp_path / d).mkdir(exist_ok=True) - files = ["cloud.yml", "tenant.yml", "bg_domain.yml", "composite_structure.yml"] - for f in files: - (tmp_path / f).write_text("text") - (tmp_path / "keep.yml").write_text("text") - (tmp_path / "keep").mkdir(exist_ok=True) - return tmp_path +class TestEnvBuild(BaseTest): + @pytest.fixture(autouse=True) + def change_test_dir(self, monkeypatch): + monkeypatch.chdir(self.base_dir) -def test_cleanup_target_dir_removes_expected_items(): - target_dir = Path(g_output_dir) / "dump-cluster" - setup_test_dir(target_dir) - cleanup_resulting_dir(Path(target_dir)) - assert not (target_dir / "Applications").exists() - assert not (target_dir / "Namespaces").exists() - assert not (target_dir / "Profiles").exists() - assert not (target_dir / "cloud.yml").exists() - assert not (target_dir / "tenant.yml").exists() - assert not (target_dir / "bg_domain.yml").exists() - assert not (target_dir / "composite_structure.yml").exists() + @pytest.mark.parametrize("cluster_name, env_name, version", test_data) + def test_render_envs(self, cluster_name, env_name, version): + g_templates_dir = str((self.test_data_dir / "test_templates").resolve()) + g_inventory_dir = str((self.test_data_dir / "test_environments").resolve()) + g_output_dir = str((self.base_dir / "/tmp/test_environments").resolve()) - assert (target_dir / "keep.yml").exists() - assert (target_dir / "keep").exists() + os.environ['CI_COMMIT_REF_NAME'] = "branch_name" + environ['FULL_ENV_NAME'] = cluster_name + '/' + env_name - delete_dir(target_dir) + render_environment(env_name, cluster_name, g_templates_dir, g_inventory_dir, g_output_dir, self.test_data_dir) + source_dir = f"{g_inventory_dir}/{cluster_name}/{env_name}" + generated_dir = f"{g_output_dir}/{cluster_name}/{env_name}" + files_to_compare = get_all_files_in_dir(source_dir) + logger.info(dump_as_yaml_format(files_to_compare)) + TestHelpers.assert_dirs_content(source_dir, generated_dir, True, False) diff --git a/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py new file mode 100644 index 000000000..871fe92ed --- /dev/null +++ b/scripts/build_env/tests/env_inventory_generation/test_env_inv_generation.py @@ -0,0 +1,114 @@ +from os import environ +from pathlib import Path + +import yaml +from env_inventory_generation import generate_env_new_approach, Place, resolve_path, INVENTORY, Action +from envgenehelper import get_cluster_name_from_full_name, dumpYamlToStr, get_environment_name_from_full_name, readYaml, \ + is_dir_empty, writeYamlToFile +from envgenehelper.test_helpers import TestHelpers +from jinja.jinja import create_jinja_env +from tests.base_test import BaseTest + +FEATURE_TEST_DIR = "test_inventory_generation" + + +class TestEnvInvGen(BaseTest): + full_env_name = "cluster-01/env-01" + env_dir = "" + site_dir = "" + cluster_dir = "" + config_dir = "" + action: Action = None + + def setup_method(self): + self.set_ci_project_dir(FEATURE_TEST_DIR, "output") + self.expected_dir = self.expected_dir.joinpath(FEATURE_TEST_DIR, "expected") + TestHelpers.clean_test_dir(self.ci_project_dir) + + self.env_name = get_environment_name_from_full_name(self.full_env_name) + self.cluster = get_cluster_name_from_full_name(self.full_env_name) + + environ["ENV_NAME"] = self.env_name + environ["CLUSTER_NAME"] = self.cluster + environ["FULL_ENV_NAME"] = self.full_env_name + + site = Path(self.ci_project_dir) / "environments" + self.site_dir = site + self.cluster_dir = self.site_dir / self.cluster + self.env_dir = site / self.full_env_name + + self.config_dir = self.ci_project_dir / "configuration" / "config.yml" + writeYamlToFile(self.config_dir, yaml.safe_load("crypt: false\n")) + + def set_inv_content(self): + places = [p.value for p in Place] + template = create_jinja_env(self.test_data_dir / FEATURE_TEST_DIR / "input").get_template("content.yml.j2") + content = template.render(places=places, action=self.action.value, env=self.env_name, cluster=self.cluster) + environ["ENV_INVENTORY_CONTENT"] = content + + def test_resolve_path(self): + env_dir = Path("/repo/environments/cluster-01/env-01") + cases = [ + (Place.ENV, "parameters", INVENTORY, "ps1", + "/repo/environments/cluster-01/env-01/Inventory/parameters/ps1.yml"), + (Place.CLUSTER, "parameters", INVENTORY, "ps1", "/repo/environments/cluster-01/parameters/ps1.yml"), + (Place.SITE, "parameters", INVENTORY, "ps1", "/repo/environments/parameters/ps1.yml"), + (Place.ENV, "credentials", INVENTORY, "cred1", + "/repo/environments/cluster-01/env-01/Inventory/credentials/cred1.yml"), + (Place.CLUSTER, "credentials", INVENTORY, "cred1", "/repo/environments/cluster-01/credentials/cred1.yml"), + (Place.SITE, "credentials", INVENTORY, "cred1", "/repo/environments/credentials/cred1.yml"), + (Place.ENV, "resource_profiles", INVENTORY, "rp1", + "/repo/environments/cluster-01/env-01/Inventory/resource_profiles/rp1.yml"), + (Place.CLUSTER, "resource_profiles", INVENTORY, "rp1", + "/repo/environments/cluster-01/resource_profiles/rp1.yml"), + (Place.SITE, "resource_profiles", INVENTORY, "rp1", "/repo/environments/resource_profiles/rp1.yml"), + (Place.ENV, "shared_template_variables", "", "stv1", + "/repo/environments/cluster-01/env-01/shared_template_variables/stv1.yml"), + (Place.CLUSTER, "shared_template_variables", "", "stv1", + "/repo/environments/cluster-01/shared_template_variables/stv1.yml"), + (Place.SITE, "shared_template_variables", "", "stv1", + "/repo/environments/shared_template_variables/stv1.yml"), + ] + for place, subdir, inventory, name, expected in cases: + result_path = resolve_path(env_dir, place, subdir, name, inventory) + assert result_path == Path(expected) + + def test_gen_env_create_delete(self): + self.action = Action.CREATE_OR_REPLACE + self.set_inv_content() + + generate_env_new_approach() + + TestHelpers.assert_dirs_content(self.ci_project_dir, self.expected_dir, check_for_extra_files=True, + check_for_missing_files=True) + + self.action = Action.DELETE + self.set_inv_content() + + generate_env_new_approach() + + assert is_dir_empty(self.site_dir / "parameters") + assert is_dir_empty(self.site_dir / "credentials") + assert is_dir_empty(self.site_dir / "resource_profiles") + assert is_dir_empty(self.site_dir / "shared_template_variables") + + assert self.cluster_dir.exists() + assert is_dir_empty(self.cluster_dir / "parameters") + assert is_dir_empty(self.cluster_dir / "credentials") + assert is_dir_empty(self.cluster_dir / "resource_profiles") + assert is_dir_empty(self.cluster_dir / "shared_template_variables") + + assert not self.env_dir.exists() + + def test_env_template_version(self): + self.action = Action.CREATE_OR_REPLACE + self.set_inv_content() + updated_version = "template:v0.0.0" + environ["ENV_TEMPLATE_VERSION"] = updated_version + + generate_env_new_approach() + + inventory_path = Path( + self.ci_project_dir) / "environments" / self.full_env_name / INVENTORY / "env_definition.yml" + inventory = readYaml(inventory_path, safe_load=True) + assert inventory["envTemplate"]["artifact"] == updated_version diff --git a/scripts/utils/pipeline_parameters.py b/scripts/utils/pipeline_parameters.py index 9617f67f9..e943febf5 100644 --- a/scripts/utils/pipeline_parameters.py +++ b/scripts/utils/pipeline_parameters.py @@ -21,7 +21,7 @@ def get_pipeline_parameters() -> dict: "SD_DELTA": getenv("SD_DELTA"), "SD_REPO_MERGE_MODE": getenv("SD_REPO_MERGE_MODE"), "ENV_INVENTORY_INIT": getenv("ENV_INVENTORY_INIT"), - "ENV_SPECIFIC_PARAMETERS": getenv("ENV_SPECIFIC_PARAMS"), + "ENV_SPECIFIC_PARAMS": getenv("ENV_SPECIFIC_PARAMS"), "ENV_TEMPLATE_NAME": getenv("ENV_TEMPLATE_NAME"), 'CRED_ROTATION_PAYLOAD': getenv("CRED_ROTATION_PAYLOAD", ""), 'CRED_ROTATION_FORCE': getenv("CRED_ROTATION_FORCE", ""), @@ -32,6 +32,7 @@ def get_pipeline_parameters() -> dict: 'ENVGENE_LOG_LEVEL': getenv("ENVGENE_LOG_LEVEL"), "BG_STATE": getenv("BG_STATE", None), "BG_MANAGE": getenv("BG_MANAGE", None) == "true", + "ENV_INVENTORY_CONTENT": getenv("ENV_INVENTORY_CONTENT") } class PipelineParametersHandler: diff --git a/test_data/test_app_reg_defs/TC-001-001/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-001/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-001/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_environments/bgd-cluster/bgd-env/AppDefs/application-1.yml b/test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yaml similarity index 100% rename from test_data/test_environments/bgd-cluster/bgd-env/AppDefs/application-1.yml rename to test_data/test_app_reg_defs/TC-001-001/expected/appdefs/application-1.yaml diff --git a/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..b70639ff3 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-001/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides.registryName | default('registry-1') }}" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-002/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-002/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-002/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-1.yml b/test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yaml similarity index 100% rename from test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-1.yml rename to test_data/test_app_reg_defs/TC-001-002/expected/regdefs/registry-1.yaml diff --git a/test_data/test_app_reg_defs/TC-001-002/templates/regdefs/registry-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-002/templates/regdefs/registry-1.yaml.j2 new file mode 100644 index 000000000..366cff847 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-002/templates/regdefs/registry-1.yaml.j2 @@ -0,0 +1,19 @@ +name: "registry-1" +credentialsId: "registry-cred" +mavenConfig: + repositoryDomainName: "{{ regdefs.overrides.maven.RepositoryDomainName | default('maven.qubership.org') }}" + fullRepositoryUrl: "{{ regdefs.overrides.maven.fullRepositoryUrl | default('https://maven.qubership.org/repository') }}" + targetSnapshot: "snapshot" + targetStaging: "staging" + targetRelease: "release" + releaseGroup: "" + snapshotGroup: "" +dockerConfig: + snapshotUri: "{{ regdefs.overrides.docker.snapshotUri | default('docker.qubership.org/snapshot') }}" + stagingUri: "{{ regdefs.overrides.docker.stagingUri | default('docker.qubership.org/staging') }}" + releaseUri: "{{ regdefs.overrides.docker.releaseUri | default('docker.qubership.org/release') }}" + groupUri: "{{ regdefs.overrides.docker.groupUri | default('docker.qubership.org/group') }}" + snapshotRepoName: "docker-snapshot" + stagingRepoName: "docker-staging" + releaseRepoName: "docker-release" + groupName: "docker-group" diff --git a/test_data/test_app_reg_defs/TC-001-003/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-003/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-003/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-003/environments/configuration/appregdef_config.yaml b/test_data/test_app_reg_defs/TC-001-003/environments/configuration/appregdef_config.yaml new file mode 100644 index 000000000..13d5b97d2 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-003/environments/configuration/appregdef_config.yaml @@ -0,0 +1,4 @@ +appdefs: + overrides: + "com.example:app1": + registryName: "custom-registry" \ No newline at end of file diff --git a/test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-1.yml b/test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml similarity index 83% rename from test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-1.yml rename to test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml index cf96b3568..f0a957a26 100644 --- a/test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-1.yml +++ b/test_data/test_app_reg_defs/TC-001-003/expected/appdefs/application-1.yaml @@ -1,5 +1,5 @@ name: "application-1" -registryName: "registry-1" +registryName: "custom-registry" artifactId: "application-1" groupId: "org.qubership" supportParallelDeploy: true diff --git a/test_data/test_app_reg_defs/TC-001-003/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-003/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..65e464851 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-003/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides["com.example:app1"].registryName | default('registry-1') }}" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-004/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-004/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-004/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-004/environments/configuration/appregdef_config.yaml b/test_data/test_app_reg_defs/TC-001-004/environments/configuration/appregdef_config.yaml new file mode 100644 index 000000000..9085ea740 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-004/environments/configuration/appregdef_config.yaml @@ -0,0 +1,5 @@ +regdefs: + overrides: + "maven-registry": + repositoryDomainName: "qubership.org" + fullRepositoryUrl: "https://qubership.org" diff --git a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml b/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml similarity index 78% rename from test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml rename to test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml index 599de06df..4f4b1d908 100644 --- a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-1.yml +++ b/test_data/test_app_reg_defs/TC-001-004/expected/regdefs/registry-1.yaml @@ -1,8 +1,8 @@ name: "registry-1" credentialsId: "registry-cred" mavenConfig: - repositoryDomainName: "maven.qubership.org" - fullRepositoryUrl: "https://maven.qubership.org/repository" + repositoryDomainName: "qubership.org" + fullRepositoryUrl: "https://qubership.org" targetSnapshot: "snapshot" targetStaging: "staging" targetRelease: "release" @@ -16,4 +16,4 @@ dockerConfig: snapshotRepoName: "docker-snapshot" stagingRepoName: "docker-staging" releaseRepoName: "docker-release" - groupName: "docker-group" + groupName: "docker-group" \ No newline at end of file diff --git a/test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-1.yml b/test_data/test_app_reg_defs/TC-001-004/templates/regdefs/registry-1.yaml.j2 similarity index 63% rename from test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-1.yml rename to test_data/test_app_reg_defs/TC-001-004/templates/regdefs/registry-1.yaml.j2 index 599de06df..1d580f7f6 100644 --- a/test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-1.yml +++ b/test_data/test_app_reg_defs/TC-001-004/templates/regdefs/registry-1.yaml.j2 @@ -1,8 +1,8 @@ name: "registry-1" credentialsId: "registry-cred" mavenConfig: - repositoryDomainName: "maven.qubership.org" - fullRepositoryUrl: "https://maven.qubership.org/repository" + repositoryDomainName: "{{ regdefs.overrides['maven-registry'].repositoryDomainName | default('maven.qubership.org') }}" + fullRepositoryUrl: "{{ regdefs.overrides['maven-registry'].fullRepositoryUrl | default('https://maven.qubership.org/repository') }}" targetSnapshot: "snapshot" targetStaging: "staging" targetRelease: "release" @@ -16,4 +16,4 @@ dockerConfig: snapshotRepoName: "docker-snapshot" stagingRepoName: "docker-staging" releaseRepoName: "docker-release" - groupName: "docker-group" + groupName: "docker-group" \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/configuration/appregdef_config.yaml b/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/configuration/appregdef_config.yaml new file mode 100644 index 000000000..13d5b97d2 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/configuration/appregdef_config.yaml @@ -0,0 +1,4 @@ +appdefs: + overrides: + "com.example:app1": + registryName: "custom-registry" \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-005/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml new file mode 100644 index 000000000..f0a957a26 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-005/expected/appdefs/application-1.yaml @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "custom-registry" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-005/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-005/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..65e464851 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-005/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides["com.example:app1"].registryName | default('registry-1') }}" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/configuration/appregdef_config.yaml b/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/configuration/appregdef_config.yaml new file mode 100644 index 000000000..e8563a58b --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/configuration/appregdef_config.yaml @@ -0,0 +1,4 @@ +appdefs: + overrides: + "com.example:app1": + supportParallelDeploy: true \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-006/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-006/environments/configuration/appregdef_config.yaml b/test_data/test_app_reg_defs/TC-001-006/environments/configuration/appregdef_config.yaml new file mode 100644 index 000000000..9b960a988 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-006/environments/configuration/appregdef_config.yaml @@ -0,0 +1,5 @@ +appdefs: + overrides: + "com.example:app1": + registryName: "global-registry" + supportParallelDeploy: false \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml b/test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml new file mode 100644 index 000000000..2b8d97519 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-006/expected/appdefs/application-1.yaml @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "global-registry" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-006/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-006/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..800b9946a --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-006/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides["com.example:app1"].registryName | default('registry-1') }}" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: {{ appdefs.overrides["com.example:app1"].supportParallelDeploy | default(false) }} +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-008/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-008/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-008/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_environments/cluster-01/env-01/AppDefs/application-1.yml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml similarity index 100% rename from test_data/test_environments/cluster-01/env-01/AppDefs/application-1.yml rename to test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-1.yaml diff --git a/test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2 similarity index 77% rename from test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml rename to test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2 index a65effbc6..3a1b23893 100644 --- a/test_data/test_environments/cluster-01/env-01/AppDefs/application-2.yml +++ b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-2.yml.j2 @@ -5,5 +5,3 @@ groupId: "org.qubership" supportParallelDeploy: true deployParameters: {} technicalConfigurationParameters: {} -metadata: - helmReleaseNameStrategy: "perApplication" diff --git a/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml new file mode 100644 index 000000000..53606066f --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-008/expected/appdefs/application-3.yml @@ -0,0 +1,7 @@ +name: "application-3" +registryName: "registry-3" +artifactId: "application-3" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..b70639ff3 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides.registryName | default('registry-1') }}" +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_environments/bgd-cluster/bgd-env/AppDefs/application-2.yml b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-2.yml.j2 similarity index 60% rename from test_data/test_environments/bgd-cluster/bgd-env/AppDefs/application-2.yml rename to test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-2.yml.j2 index a65effbc6..8fd52e0db 100644 --- a/test_data/test_environments/bgd-cluster/bgd-env/AppDefs/application-2.yml +++ b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-2.yml.j2 @@ -1,9 +1,7 @@ name: "application-2" -registryName: "registry-2" +registryName: "{{ appdefs.overrides['com.example:application-2'].registryName | default('registry-2') }}" artifactId: "application-2" groupId: "org.qubership" supportParallelDeploy: true deployParameters: {} technicalConfigurationParameters: {} -metadata: - helmReleaseNameStrategy: "perApplication" diff --git a/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-3.yml b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-3.yml new file mode 100644 index 000000000..0c61a8281 --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-008/templates/appdefs/application-3.yml @@ -0,0 +1,7 @@ +name: "application-3" +registryName: "{{ appdefs.overrides['com.example:application-3'].registryName | default('registry-3') }}" +artifactId: "application-3" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} \ No newline at end of file diff --git a/test_data/test_app_reg_defs/TC-001-010/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-010/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-010/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-010/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-010/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..b8e8fa68d --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-010/templates/appdefs/application-1.yaml.j2 @@ -0,0 +1,7 @@ +name: "application-1" +registryName: "{{ appdefs.overrides.registryName | default('registry-1') " +artifactId: "application-1" +groupId: "org.qubership" +supportParallelDeploy: true +deployParameters: {} +technicalConfigurationParameters: {} diff --git a/test_data/test_app_reg_defs/TC-001-011/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-011/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-011/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-011/templates/appdefs/application-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-011/templates/appdefs/application-1.yaml.j2 new file mode 100644 index 000000000..e69de29bb diff --git a/test_data/test_app_reg_defs/TC-001-012/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_app_reg_defs/TC-001-012/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..b13b9aade --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-012/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: env-01 + tenantName: tenant + cloudName: cluster_01_env_01 + cloudPassport: cluster-01 + description: Composite Full Sample + owners: Qubership team +envTemplate: + name: composite-prod + artifact: deployment-configuration-env-templates:1.2.3 + additionalTemplateVariables: + ci: + CI_PARAM_1: ci-param-val-1 + CI_PARAM_2: ci-param-val-2 + e2eParameters: + E2E_PARAM_1: e2e-param-val-1 + E2E_PARAM_2: e2e-param-val-2 + cleanInstallApprovalRequired: true + sharedTemplateVariables: + - prod-template-variables + - sample-cloud-template-variables + envSpecificParamsets: + bss: + - env-specific-bss + - prod-shared + envSpecificTechnicalParamsets: + bss: + - env-specific-tech + envSpecificE2EParamsets: + cloud: + - cloud-level-params + sharedMasterCredentialFiles: + - integration-creds +generatedVersions: + generateEnvironmentLatestVersion: "deployment-configuration-env-templates:composite-prod" # This value is automatically generated during job run. diff --git a/test_data/test_app_reg_defs/TC-001-012/templates/regdefs/registry-1.yaml.j2 b/test_data/test_app_reg_defs/TC-001-012/templates/regdefs/registry-1.yaml.j2 new file mode 100644 index 000000000..308cee22e --- /dev/null +++ b/test_data/test_app_reg_defs/TC-001-012/templates/regdefs/registry-1.yaml.j2 @@ -0,0 +1,18 @@ +credentialsId: "registry-cred" +mavenConfig: + repositoryDomainName: "{{ regdefs.overrides.maven.RepositoryDomainName | default('maven.qubership.org') }}" + fullRepositoryUrl: "{{ regdefs.overrides.maven.fullRepositoryUrl | default('https://maven.qubership.org/repository') }}" + targetSnapshot: "snapshot" + targetStaging: "staging" + targetRelease: "release" + releaseGroup: "" + snapshotGroup: "" +dockerConfig: + snapshotUri: "{{ regdefs.overrides.docker.snapshotUri | default('docker.qubership.org/snapshot') }}" + stagingUri: "{{ regdefs.overrides.docker.stagingUri | default('docker.qubership.org/staging') }}" + releaseUri: "{{ regdefs.overrides.docker.releaseUri | default('docker.qubership.org/release') }}" + groupUri: "{{ regdefs.overrides.docker.groupUri | default('docker.qubership.org/group') }}" + snapshotRepoName: "docker-snapshot" + stagingRepoName: "docker-staging" + releaseRepoName: "docker-release" + groupName: "docker-group" diff --git a/test_data/test_environments/bgd-cluster/bgd-env/Namespaces/origin-app/namespace.yml b/test_data/test_environments/bgd-cluster/bgd-env/Namespaces/app-origin/namespace.yml similarity index 100% rename from test_data/test_environments/bgd-cluster/bgd-env/Namespaces/origin-app/namespace.yml rename to test_data/test_environments/bgd-cluster/bgd-env/Namespaces/app-origin/namespace.yml diff --git a/test_data/test_environments/bgd-cluster/bgd-env/Namespaces/peer-app/namespace.yml b/test_data/test_environments/bgd-cluster/bgd-env/Namespaces/app-peer/namespace.yml similarity index 100% rename from test_data/test_environments/bgd-cluster/bgd-env/Namespaces/peer-app/namespace.yml rename to test_data/test_environments/bgd-cluster/bgd-env/Namespaces/app-peer/namespace.yml diff --git a/test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-2.yml b/test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-2.yml deleted file mode 100644 index ef136e375..000000000 --- a/test_data/test_environments/bgd-cluster/bgd-env/RegDefs/registry-2.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "registry-2" -credentialsId: "registry-cred" -mavenConfig: - repositoryDomainName: "maven.qubership.org" - fullRepositoryUrl: "https://maven.qubership.org/repository" - targetSnapshot: "snapshot" - targetStaging: "staging" - targetRelease: "release" - releaseGroup: "" - snapshotGroup: "" -dockerConfig: - snapshotUri: "docker.qubership.org/snapshot" - stagingUri: "docker.qubership.org/staging" - releaseUri: "docker.qubership.org/release" - groupUri: "docker.qubership.org/group" - snapshotRepoName: "docker-snapshot" - stagingRepoName: "docker-staging" - releaseRepoName: "docker-release" - groupName: "docker-group" diff --git a/test_data/test_environments/bgd-cluster/bgd-env/bg_domain.yml b/test_data/test_environments/bgd-cluster/bgd-env/bg_domain.yml index 6e9446e4c..910b38a0f 100644 --- a/test_data/test_environments/bgd-cluster/bgd-env/bg_domain.yml +++ b/test_data/test_environments/bgd-cluster/bgd-env/bg_domain.yml @@ -1,10 +1,10 @@ name: "bgd-env-bg-domain" type: bgdomain originNamespace: - name: "bgd-env-origin-bss" + name: "bgd-env-origin-app" type: namespace peerNamespace: - name: "bgd-env-peer-bss" + name: "bgd-env-peer-app" type: namespace controllerNamespace: name: "bgd-env-bg-controller" diff --git a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml b/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml deleted file mode 100644 index ef136e375..000000000 --- a/test_data/test_environments/cluster-01/env-01/RegDefs/registry-2.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "registry-2" -credentialsId: "registry-cred" -mavenConfig: - repositoryDomainName: "maven.qubership.org" - fullRepositoryUrl: "https://maven.qubership.org/repository" - targetSnapshot: "snapshot" - targetStaging: "staging" - targetRelease: "release" - releaseGroup: "" - snapshotGroup: "" -dockerConfig: - snapshotUri: "docker.qubership.org/snapshot" - stagingUri: "docker.qubership.org/staging" - releaseUri: "docker.qubership.org/release" - groupUri: "docker.qubership.org/group" - snapshotRepoName: "docker-snapshot" - stagingRepoName: "docker-staging" - releaseRepoName: "docker-release" - groupName: "docker-group" diff --git a/test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-2.yml b/test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-2.yml deleted file mode 100644 index a65effbc6..000000000 --- a/test_data/test_environments/cluster03/rpo-replacement-mode/AppDefs/application-2.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: "application-2" -registryName: "registry-2" -artifactId: "application-2" -groupId: "org.qubership" -supportParallelDeploy: true -deployParameters: {} -technicalConfigurationParameters: {} -metadata: - helmReleaseNameStrategy: "perApplication" diff --git a/test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-2.yml b/test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-2.yml deleted file mode 100644 index ef136e375..000000000 --- a/test_data/test_environments/cluster03/rpo-replacement-mode/RegDefs/registry-2.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "registry-2" -credentialsId: "registry-cred" -mavenConfig: - repositoryDomainName: "maven.qubership.org" - fullRepositoryUrl: "https://maven.qubership.org/repository" - targetSnapshot: "snapshot" - targetStaging: "staging" - targetRelease: "release" - releaseGroup: "" - snapshotGroup: "" -dockerConfig: - snapshotUri: "docker.qubership.org/snapshot" - stagingUri: "docker.qubership.org/staging" - releaseUri: "docker.qubership.org/release" - groupUri: "docker.qubership.org/group" - snapshotRepoName: "docker-snapshot" - stagingRepoName: "docker-staging" - releaseRepoName: "docker-release" - groupName: "docker-group" diff --git a/test_data/test_inventory_generation/expected/configuration/config.yml b/test_data/test_inventory_generation/expected/configuration/config.yml new file mode 100644 index 000000000..b221e6e45 --- /dev/null +++ b/test_data/test_inventory_generation/expected/configuration/config.yml @@ -0,0 +1 @@ +crypt: false diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/credentials/shared-credentials-cluster.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/credentials/shared-credentials-cluster.yml new file mode 100644 index 000000000..4d854c5d5 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/credentials/shared-credentials-cluster.yml @@ -0,0 +1,5 @@ +prod-integration-creds: + type: "usernamePassword" + data: + username: "" + password: "" diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/credentials/shared-credentials-env.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/credentials/shared-credentials-env.yml new file mode 100644 index 000000000..4d854c5d5 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/credentials/shared-credentials-env.yml @@ -0,0 +1,5 @@ +prod-integration-creds: + type: "usernamePassword" + data: + username: "" + password: "" diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/env_definition.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/env_definition.yml new file mode 100644 index 000000000..1105af2da --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/env_definition.yml @@ -0,0 +1,35 @@ +inventory: + environmentName: "env-01" + tenantName: "Applications" + cloudName: "cluster-01" + description: "Full sample" + owners: "Qubership team" + config: + updateRPOverrideNameWithEnvName: false + updateCredIdsWithEnvName: true +envTemplate: + name: "composite-prod" + artifact: "project-env-template:master_20231024-080204" + additionalTemplateVariables: + ci: + CI_PARAM_1: "ci-param-val-1" + CI_PARAM_2: "ci-param-val-2" + e2eParameters: + E2E_PARAM_1: "e2e-param-val-1" + E2E_PARAM_2: "e2e-param-val-2" + sharedTemplateVariables: + - "prod-template-variables" + - "sample-cloud-template-variables" + envSpecificParamsets: + bss: + - "env-specific-bss" + envSpecificTechnicalParamsets: + bss: + - "env-specific-tech" + envSpecificE2EParamsets: + cloud: + - "cloud-level-params" + sharedMasterCredentialFiles: + - "prod-integration-creds" + envSpecificResourceProfiles: + cloud: "cloud-specific-profile" diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/parameters/env-specific-bss-env.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/parameters/env-specific-bss-env.yml new file mode 100644 index 000000000..3541bf812 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/parameters/env-specific-bss-env.yml @@ -0,0 +1,5 @@ +version: 1 +name: "env-specific-bss-env" +parameters: + key: "value" +applications: [] diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/resource_profiles/cloud-specific-profile-env.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/resource_profiles/cloud-specific-profile-env.yml new file mode 100644 index 000000000..834f71d05 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/Inventory/resource_profiles/cloud-specific-profile-env.yml @@ -0,0 +1,15 @@ +name: "cloud-specific-profile-env" +baseline: "dev" +description: "" +applications: + - name: "core" + version: "release-20241103.225817" + sd: "" + services: + - name: "operator" + parameters: + - name: "GATEWAY_MEMORY_LIMIT" + value: "96Mi" + - name: "GATEWAY_CPU_REQUEST" + value: "50m" +version: 0 diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/shared_template_variables/prod-template-variables-env.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/shared_template_variables/prod-template-variables-env.yml new file mode 100644 index 000000000..2700fba93 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/env-01/shared_template_variables/prod-template-variables-env.yml @@ -0,0 +1,5 @@ +TEMPLATE_VAR_1: "prod-value-1" +TEMPLATE_VAR_2: "prod-value-2" +nested: + key1: "nested-prod-value-1" + key2: "nested-prod-value-2" diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/parameters/env-specific-bss-cluster.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/parameters/env-specific-bss-cluster.yml new file mode 100644 index 000000000..f23d3b3a1 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/parameters/env-specific-bss-cluster.yml @@ -0,0 +1,5 @@ +version: 1 +name: "env-specific-bss-cluster" +parameters: + key: "value" +applications: [] diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/resource_profiles/cloud-specific-profile-cluster.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/resource_profiles/cloud-specific-profile-cluster.yml new file mode 100644 index 000000000..b650fc599 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/resource_profiles/cloud-specific-profile-cluster.yml @@ -0,0 +1,15 @@ +name: "cloud-specific-profile-cluster" +baseline: "dev" +description: "" +applications: + - name: "core" + version: "release-20241103.225817" + sd: "" + services: + - name: "operator" + parameters: + - name: "GATEWAY_MEMORY_LIMIT" + value: "96Mi" + - name: "GATEWAY_CPU_REQUEST" + value: "50m" +version: 0 diff --git a/test_data/test_inventory_generation/expected/environments/cluster-01/shared_template_variables/prod-template-variables-cluster.yml b/test_data/test_inventory_generation/expected/environments/cluster-01/shared_template_variables/prod-template-variables-cluster.yml new file mode 100644 index 000000000..2700fba93 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/cluster-01/shared_template_variables/prod-template-variables-cluster.yml @@ -0,0 +1,5 @@ +TEMPLATE_VAR_1: "prod-value-1" +TEMPLATE_VAR_2: "prod-value-2" +nested: + key1: "nested-prod-value-1" + key2: "nested-prod-value-2" diff --git a/test_data/test_inventory_generation/expected/environments/credentials/shared-credentials-site.yml b/test_data/test_inventory_generation/expected/environments/credentials/shared-credentials-site.yml new file mode 100644 index 000000000..4d854c5d5 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/credentials/shared-credentials-site.yml @@ -0,0 +1,5 @@ +prod-integration-creds: + type: "usernamePassword" + data: + username: "" + password: "" diff --git a/test_data/test_inventory_generation/expected/environments/parameters/env-specific-bss-site.yml b/test_data/test_inventory_generation/expected/environments/parameters/env-specific-bss-site.yml new file mode 100644 index 000000000..3261674b3 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/parameters/env-specific-bss-site.yml @@ -0,0 +1,5 @@ +version: 1 +name: "env-specific-bss-site" +parameters: + key: "value" +applications: [] diff --git a/test_data/test_inventory_generation/expected/environments/resource_profiles/cloud-specific-profile-site.yml b/test_data/test_inventory_generation/expected/environments/resource_profiles/cloud-specific-profile-site.yml new file mode 100644 index 000000000..51427e9ad --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/resource_profiles/cloud-specific-profile-site.yml @@ -0,0 +1,15 @@ +name: "cloud-specific-profile-site" +baseline: "dev" +description: "" +applications: + - name: "core" + version: "release-20241103.225817" + sd: "" + services: + - name: "operator" + parameters: + - name: "GATEWAY_MEMORY_LIMIT" + value: "96Mi" + - name: "GATEWAY_CPU_REQUEST" + value: "50m" +version: 0 diff --git a/test_data/test_inventory_generation/expected/environments/shared_template_variables/prod-template-variables-site.yml b/test_data/test_inventory_generation/expected/environments/shared_template_variables/prod-template-variables-site.yml new file mode 100644 index 000000000..2700fba93 --- /dev/null +++ b/test_data/test_inventory_generation/expected/environments/shared_template_variables/prod-template-variables-site.yml @@ -0,0 +1,5 @@ +TEMPLATE_VAR_1: "prod-value-1" +TEMPLATE_VAR_2: "prod-value-2" +nested: + key1: "nested-prod-value-1" + key2: "nested-prod-value-2" diff --git a/test_data/test_inventory_generation/input/content.yml.j2 b/test_data/test_inventory_generation/input/content.yml.j2 new file mode 100644 index 000000000..e247e298c --- /dev/null +++ b/test_data/test_inventory_generation/input/content.yml.j2 @@ -0,0 +1,144 @@ +{ + "envDefinition": { + "action": "{{ action }}", + "content": { + "inventory": { + "environmentName": "{{ env }}", + "tenantName": "Applications", + "cloudName": "{{ cluster }}", + "description": "Full sample", + "owners": "Qubership team", + "config": { + "updateRPOverrideNameWithEnvName": false, + "updateCredIdsWithEnvName": true + } + }, + "envTemplate": { + "name": "composite-prod", + "artifact": "project-env-template:master_20231024-080204", + "additionalTemplateVariables": { + "ci": { + "CI_PARAM_1": "ci-param-val-1", + "CI_PARAM_2": "ci-param-val-2" + }, + "e2eParameters": { + "E2E_PARAM_1": "e2e-param-val-1", + "E2E_PARAM_2": "e2e-param-val-2" + } + }, + "sharedTemplateVariables": [ + "prod-template-variables", + "sample-cloud-template-variables" + ], + "envSpecificParamsets": { + "bss": [ + "env-specific-bss" + ] + }, + "envSpecificTechnicalParamsets": { + "bss": [ + "env-specific-tech" + ] + }, + "envSpecificE2EParamsets": { + "cloud": [ + "cloud-level-params" + ] + }, + "sharedMasterCredentialFiles": [ + "prod-integration-creds" + ], + "envSpecificResourceProfiles": { + "cloud": "cloud-specific-profile" + } + } + } + }, + "paramSets": [ + {% for place in places %} + { + "action": "{{ action }}", + "place": "{{ place }}", + "content": { + "version": 1, + "name": "env-specific-bss-{{ place }}", + "parameters": { + "key": "value" + }, + "applications": [] + } + }{% if not loop.last %},{% endif %} + {% endfor %} + ], + "credentials": [ + {% for place in places %} + { + "action": "{{ action }}", + "place": "{{ place }}", + "name": "shared-credentials-{{ place }}", + "content": { + "prod-integration-creds": { + "type": "usernamePassword", + "data": { + "username": "", + "password": "" + } + } + } + }{% if not loop.last %},{% endif %} + {% endfor %} + ], + "resourceProfiles": [ + {% for place in places %} + { + "action": "{{ action }}", + "place": "{{ place }}", + "content": { + "name": "cloud-specific-profile-{{ place }}", + "baseline": "dev", + "description": "", + "applications": [ + { + "name": "core", + "version": "release-20241103.225817", + "sd": "", + "services": [ + { + "name": "operator", + "parameters": [ + { + "name": "GATEWAY_MEMORY_LIMIT", + "value": "96Mi" + }, + { + "name": "GATEWAY_CPU_REQUEST", + "value": "50m" + } + ] + } + ] + } + ], + "version": 0 + } + }{% if not loop.last %},{% endif %} + {% endfor %} + ], + "sharedTemplateVariables": [ + {% for place in places %} + { + "action": "{{ action }}", + "place": "{{ place }}", + "name": "prod-template-variables-{{ place }}", + "content": { + "TEMPLATE_VAR_1": "prod-value-1", + "TEMPLATE_VAR_2": "prod-value-2", + "nested": { + "key1": "nested-prod-value-1", + "key2": "nested-prod-value-2" + } + } + }{% if not loop.last %},{% endif %} + {% endfor %} + ] +} diff --git a/test_data/test_templates/env_templates/bgd.yaml b/test_data/test_templates/env_templates/bgd.yaml index a1af60968..ab25e07c6 100644 --- a/test_data/test_templates/env_templates/bgd.yaml +++ b/test_data/test_templates/env_templates/bgd.yaml @@ -4,13 +4,11 @@ cloud: template_path: "{{ templates_dir }}/env_templates/bgd/cloud.yml.j2" namespaces: - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/app.yml.j2" - deploy_postfix: "peer-app" template_override: - name: "{{current_env.name}}-peer-app" + name: "{{ current_env.name }}-peer-app" - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/app.yml.j2" - deploy_postfix: "origin-app" template_override: - name: "{{current_env.name}}-origin-app" + name: "{{ current_env.name }}-origin-app" - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-plugin.yml.j2" - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/bg-controller.yml.j2" - template_path: "{{ templates_dir }}/env_templates/bgd/Namespaces/other-app.yml.j2" diff --git a/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 b/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 index a239f7738..6c44323f4 100644 --- a/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 +++ b/test_data/test_templates/env_templates/bgd/bg_domain.yml.j2 @@ -1,13 +1,13 @@ name: "{{current_env.name}}-bg-domain" type: bgdomain originNamespace: - name: "{{current_env.name}}-origin-bss" + name: "{{current_env.name}}-origin-app" type: namespace peerNamespace: - name: "{{current_env.name}}-peer-bss" + name: "{{current_env.name}}-peer-app" type: namespace controllerNamespace: name: "{{current_env.name}}-bg-controller" type: namespace credentials: bgdomaincred - + From 452b41c81dac49df88ab154e2bbd0f0e83b2c5a1 Mon Sep 17 00:00:00 2001 From: GeethaGadde99 Date: Wed, 4 Feb 2026 11:53:33 +0530 Subject: [PATCH 4/4] feat: Enhance Test data for Calculator CLI --- .../pl-01/Namespaces/monitoring-origin/namespace.yml | 4 ++++ .../cleanup/monitoring-origin/parameters.yaml | 4 ++++ .../MONITORING/values/deployment-parameters.yaml | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml index f00fc37a0..9bcb4ddc9 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/Namespaces/monitoring-origin/namespace.yml @@ -17,6 +17,10 @@ deployParameters: app_version: "3.0" ssl_enabled: true debug_mode_test: "true" + api_port: ${server_port} + service_version: ${app_version} + use_ssl: ${ssl_enabled} + log_level: ${debug_mode_test} ENVGENE_CONFIG_REF_NAME: "branch_name" ENVGENE_CONFIG_TAG: "No Ref tag" KMS_CERT_IN_BASE64: "${creds.get( \"kms-cert\" ).secret}" # paramset: test-deploy-creds version: 1 source: template diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml index 66dd2aa24..3b281204b 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/cleanup/monitoring-origin/parameters.yaml @@ -77,6 +77,7 @@ api_config: connection: host: db.example.com port: 5432 +api_port: 8080 app_version: '3.0' bss-app-exist: false core: @@ -89,6 +90,7 @@ database_config: host: db.example.com port: 5432 debug_mode_test: 'true' +log_level: 'true' rendered_template: |- services: api: @@ -96,7 +98,9 @@ rendered_template: |- ports: - 8080:8080 server_port: 8080 +service_version: '3.0' ssl_enabled: true +use_ssl: true yaml_template: |- services: api: diff --git a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml index 66fd83c5d..2bf96010b 100644 --- a/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml +++ b/build_effective_set_generator/effective-set-generator/src/test/resources/environments/cluster-01/pl-01/effective-set/deployment/monitoring-origin/MONITORING/values/deployment-parameters.yaml @@ -78,6 +78,7 @@ api_config: &id001 connection: host: db.example.com port: 5432 +api_port: 8080 app_version: '3.0' bss-app-exist: false core: &id002 @@ -90,6 +91,7 @@ database_config: &id003 host: db.example.com port: 5432 debug_mode_test: 'true' +log_level: 'true' rendered_template: |- services: api: @@ -97,7 +99,9 @@ rendered_template: |- ports: - 8080:8080 server_port: 8080 +service_version: '3.0' ssl_enabled: true +use_ssl: true yaml_template: |- services: api: @@ -182,11 +186,13 @@ global: &id004 ZOOKEEPER_ADDRESS: zookeeper.zookeeper:2181 ZOOKEEPER_URL: zookeeper.zookeeper:2181 api_config: *id001 + api_port: 8080 app_version: '3.0' bss-app-exist: false core: *id002 database_config: *id003 debug_mode_test: 'true' + log_level: 'true' rendered_template: |- services: api: @@ -194,7 +200,9 @@ global: &id004 ports: - 8080:8080 server_port: 8080 + service_version: '3.0' ssl_enabled: true + use_ssl: true yaml_template: |- services: api: