diff --git a/base_modules/scripts/decrypt_fernet.py b/base_modules/scripts/decrypt_fernet.py index 86aa61192..9d6d60f1c 100644 --- a/base_modules/scripts/decrypt_fernet.py +++ b/base_modules/scripts/decrypt_fernet.py @@ -16,7 +16,6 @@ def cmdb_prepare(): @click.option('--secret_key', '-s', 'secret_key', required=True, help="Set secret_key for encrypt cred files") def decrypt_file(secret_key, file_path): - ''' {getenv('CI_PROJECT_DIR')}/ansible/inventory/group_vars/{getenv('env_name')}/appdeployer_cmdb/Tenants/{getenv('tenant_name')}/Credentials''' logger.debug('Try to read %s file', file_path) with open(file_path, mode="r", encoding="utf-8") as sensitive: sensitive_data = safe_load(sensitive) diff --git a/build_envgene/ansible/ansible.cfg b/build_envgene/ansible/ansible.cfg deleted file mode 100644 index d0c087556..000000000 --- a/build_envgene/ansible/ansible.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[defaults] -callbacks_enabled = ansible.posix.timer,ansible.posix.profile_tasks -force_color = 1 -host_key_checking = False -local_tmp = /module/ansible/tmp -retry_files_enabled = False -roles_path = /module/ansible/roles -collections_paths = /module/ansible/collections -stdout_callback = ansible.posix.debug -timeout = 300 -filter_plugins = /module/ansible/filter_plugins - - -[galaxy] -cache_dir=/module/ansible/galaxy_cache -token_path=/module/ansible/galaxy_token - -[ssh_connection] -pipelining = true -retries = 7 -ssh_args = -o ControlMaster=auto -oConnectTimeout=30 -o ControlPersist=60s -C -o PreferredAuthentications=publickey,password diff --git a/build_envgene/ansible/playbooks/git_commit.yaml b/build_envgene/ansible/playbooks/git_commit.yaml deleted file mode 100644 index 99ee59051..000000000 --- a/build_envgene/ansible/playbooks/git_commit.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Commit and push changes to git - hosts: localhost - connection: local - gather_facts: false - roles: - - role: git_commit diff --git a/build_envgene/ansible/roles/git_commit/defaults/main.yaml b/build_envgene/ansible/roles/git_commit/defaults/main.yaml deleted file mode 100644 index 6cf3014fd..000000000 --- a/build_envgene/ansible/roles/git_commit/defaults/main.yaml +++ /dev/null @@ -1,18 +0,0 @@ -### default paths -artifact_dest: /tmp/artifact.zip -build_env_path: /build_env -envgen_debug: "{{ lookup('env', 'envgen_debug') }}" - -env_name: "{{ lookup('env', 'ENV_NAME') }}" -cluster_name: "{{ lookup('env', 'CLUSTER_NAME') }}" -environment_name: "{{ lookup('env', 'ENVIRONMENT_NAME') }}" -base_dir: "{{ lookup('env', 'CI_PROJECT_DIR') }}" -env_template_vers: "{{ lookup('env', 'ENV_TEMPLATE_VERSION') }}" - -### configuration files -envs_directory_path: "{{ base_dir }}/environments" -registry_config_path: "{{ base_dir }}/configuration/registry.yml" -cred_config_path: "{{ base_dir }}/configuration/credentials/credentials.yml" - -instance_secret_key: "{{ lookup('env', 'SECRET_KEY') }}" -COMMIT_ENV: "{{ lookup('env', 'COMMIT_ENV)') }}" diff --git a/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml b/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml deleted file mode 100644 index b334cdfb8..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/01_prepare_vars.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Set env_definition path - set_fact: - env_definition_path: "{{ envs_directory_path + '/' + cluster_name + '/' + environment_name }}" diff --git a/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml b/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml deleted file mode 100644 index 23fd34fc3..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/02_git_commit.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -### script arguments -- name: Set script arguments - set_fact: - script_args: "--env_definition_path={{ env_definition_path }} --version_to_add={{ env_template_vers }}" - -- name: 03.1 Commit and push changes to git - shell: | - cd ${CI_PROJECT_DIR} - . /module/scripts/git_commit.sh diff --git a/build_envgene/ansible/roles/git_commit/tasks/main.yaml b/build_envgene/ansible/roles/git_commit/tasks/main.yaml deleted file mode 100644 index 0e8145c34..000000000 --- a/build_envgene/ansible/roles/git_commit/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: 01. prepare vars - include_tasks: 01_prepare_vars.yaml - -- name: 02. git commit - include_tasks: 02_git_commit.yaml diff --git a/build_envgene/build/Dockerfile b/build_envgene/build/Dockerfile index 6857fc417..46dd349e6 100644 --- a/build_envgene/build/Dockerfile +++ b/build_envgene/build/Dockerfile @@ -27,13 +27,11 @@ RUN apk add --no-cache \ # Copy configuration files COPY build_envgene/build/pip.conf /etc/pip.conf COPY build_envgene/build/requirements.txt /build/requirements.txt -COPY build_envgene/build/requirements.yml /build/requirements.yml COPY build_envgene/build/constraint.txt /build/constraint.txt COPY creds_rotation/build/requirements.txt /build/creds_rotation_requirements.txt # Copy source code COPY python /python -COPY build_envgene/ansible /module/ansible COPY build_envgene/scripts /module/scripts COPY scripts/bg_manage /scripts/bg_manage COPY creds_rotation/scripts /module/creds_rotation_scripts @@ -44,21 +42,10 @@ COPY scripts/cloud_passport/ /cloud_passport/scripts/ COPY schemas /build_env/schemas COPY scripts/utils /module/scripts/utils -ENV ANSIBLE_LIBRARY=/module/ansible/library - # Create virtual environment and install Python packages RUN python -m venv /module/venv RUN /module/venv/bin/pip install --upgrade pip setuptools wheel RUN /module/venv/bin/pip install --no-cache-dir --retries 10 --timeout 60 -r /build/requirements.txt -# Install essential Ansible collections -# Install to virtual environment site-packages for Python module access -RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/venv/lib/python3.12/site-packages/ansible_collections -RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/venv/lib/python3.12/site-packages/ansible_collections -RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/venv/lib/python3.12/site-packages/ansible_collections -# Also install to custom location for playbook usage -RUN /module/venv/bin/ansible-galaxy collection install ansible.utils -p /module/ansible/collections -RUN /module/venv/bin/ansible-galaxy collection install ansible.posix -p /module/ansible/collections -RUN /module/venv/bin/ansible-galaxy collection install community.general -p /module/ansible/collections RUN /module/venv/bin/pip install /python/jschon-sort RUN /module/venv/bin/pip install /python/envgene @@ -77,18 +64,13 @@ RUN apk del gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev zlib-dev RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache # Remove unnecessary files from Python packages RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete -# Don't remove test directories as they might be needed by Ansible + RUN find /module/venv/lib/python3.12/site-packages -name '*.pyo' -delete RUN find /module/venv/lib/python3.12/site-packages -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true -# Remove heavy Ansible collections that are not essential (but keep ansible.posix and ansible.utils) -RUN rm -rf /module/venv/lib/python3.12/site-packages/ansible_collections/amazon /module/venv/lib/python3.12/site-packages/ansible_collections/azure /module/venv/lib/python3.12/site-packages/ansible_collections/google /module/venv/lib/python3.12/site-packages/ansible_collections/kubernetes 2>/dev/null || true -# Remove test packages that are not needed in runtime (but keep Ansible test files) -RUN rm -rf /module/venv/lib/python3.12/site-packages/pytest* /module/venv/lib/python3.12/site-packages/_pytest* 2>/dev/null || true +RUN rm -rf /module/venv/lib/python3.12/site-packages/pytest* \ + /module/venv/lib/python3.12/site-packages/_pytest* 2>/dev/null || true RUN /module/venv/bin/pip cache purge -# Verify collections are still accessible after cleanup -RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection still accessible after cleanup')" - # Set permissions RUN chmod 754 /module/scripts/* RUN chmod 754 /module/creds_rotation_scripts/* @@ -125,9 +107,6 @@ COPY --from=build /cloud_passport /cloud_passport COPY --from=build /python /python COPY --from=build /etc/pip.conf /etc/pip.conf -# Verify collections are accessible in runtime stage -RUN /module/venv/bin/python -c "import ansible_collections.ansible.posix; print('ansible.posix collection accessible in runtime')" - # Set permissions RUN chmod +x /usr/local/bin/sops @@ -147,21 +126,15 @@ RUN mkdir -p /__w/_temp/_runner_file_commands && \ # Final cleanup RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete -# Don't remove test directories as they might be needed by Ansible RUN /module/venv/bin/pip cache purge # Keep pip for runtime compatibility, but remove setuptools and wheel -RUN rm -rf /module/venv/lib/python3.12/site-packages/setuptools* /module/venv/lib/python3.12/site-packages/wheel* 2>/dev/null || true +RUN rm -rf /module/venv/lib/python3.12/site-packages/setuptools* \ + /module/venv/lib/python3.12/site-packages/wheel* 2>/dev/null || true # Set environment ENV PATH=/module/venv/bin:$PATH \ PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - ANSIBLE_LIBRARY=/module/ansible/library \ - ANSIBLE_COLLECTIONS_PATH=/module/venv/lib/python3.12/site-packages/ansible_collections:/module/ansible/collections - -# Simple root-based container for CI/CD environments -# This container runs as root to avoid permission issues in CI/CD pipelines -WORKDIR /module/ansible + PYTHONDONTWRITEBYTECODE=1 # Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ diff --git a/build_envgene/build/requirements.txt b/build_envgene/build/requirements.txt index daea4326f..5addb5d52 100644 --- a/build_envgene/build/requirements.txt +++ b/build_envgene/build/requirements.txt @@ -13,19 +13,17 @@ jsonschema==4.19.1 jmespath==1.0.1 semantic-version==2.10.0 termcolor==2.4.0 -ansible-core==2.17.12 cffi==1.16.0 click==8.1.3 deepmerge==2.0 GitPython==3.1.45 pydantic==2.10.6 +Jinja2==3.1.6 # Additional required packages platformdirs>=3.0.0 -ansible-runner==2.4.0 # Removed heavy packages: # - shyaml, yamale, prettytable (not essential) # - ruyaml (duplicate of ruamel.yaml) # - diagrams (heavy with typed-ast dependency) -# - ansible-base (replaced with ansible-core) diff --git a/build_envgene/build/requirements.yml b/build_envgene/build/requirements.yml deleted file mode 100644 index 41925492f..000000000 --- a/build_envgene/build/requirements.yml +++ /dev/null @@ -1,5 +0,0 @@ -collections: - - name: community.general - version: 7.0.1 - - name: ansible.posix - version: 1.5.4 diff --git a/build_envgene/scripts/prepare.sh b/build_envgene/scripts/prepare.sh deleted file mode 100755 index 7eb90cb44..000000000 --- a/build_envgene/scripts/prepare.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -#### input variables -# envgen_debug -# envgen_args -# module_ansible_dir -# module_ansible_cfg -# module_inventory -# CI_SERVER_URL -# GITLAB_TOKEN - -playbook_name=$1 -ansible_dir=${module_ansible_dir} - -if ${envgen_debug} ; then set -o xtrace; fi - -chmod 700 "$ansible_dir" -cd "$ansible_dir" - -export ANSIBLE_CONFIG=${module_ansible_cfg} - -#### Run ansible -echo "ansible-playbook playbooks/$playbook_name -i ${module_inventory} ${envgen_args}" -if ansible-playbook "playbooks/$playbook_name" -i "${module_inventory}" ${envgen_args}; then - status=0 -else - status=$? -fi - -mkdir -p "$CI_PROJECT_DIR/build_env/tmp" -if [ -d "/build_env/tmp" ]; then - cp -r /build_env/tmp/* "$CI_PROJECT_DIR/build_env/tmp/" || true -fi - -exit $status - diff --git a/build_pipegene/scripts/env_build_jobs.py b/build_pipegene/scripts/env_build_jobs.py index 3dbe4f66e..ddae336ec 100644 --- a/build_pipegene/scripts/env_build_jobs.py +++ b/build_pipegene/scripts/env_build_jobs.py @@ -3,11 +3,12 @@ from pipeline_helper import job_instance -def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, cluster_name, group_id, artifact_id, tags): +def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, cluster_name, group_id, artifact_id, + tags): logger.info(f'prepare env_build job for {full_env}') script = [ - '/module/scripts/handle_certs.sh', + '/module/scripts/handle_certs.sh', ] script.append('cd /build_env; python3 /build_env/scripts/build_env/main.py') @@ -15,7 +16,7 @@ def prepare_env_build_job(pipeline, is_template_test, full_env, enviroment_name, script.append('env_name=$(cat "$CI_PROJECT_DIR/set_variable.txt")') script.append( 'sed -i "s|\\\"envgeneNullValue\\\"|\\\"test_value\\\"|g" "$CI_PROJECT_DIR/environments/$env_name/Credentials/credentials.yml"') - + env_build_params = { "name": f'env_builder.{full_env}', "image": '${envgen_image}', @@ -96,12 +97,12 @@ def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, de "stage": 'git_commit', "script": [ '/module/scripts/handle_certs.sh', - '/module/scripts/prepare.sh "git_commit.yaml"', + '/module/scripts/git_commit.sh', "export env_name=$(echo $ENV_NAME | awk -F '/' '{print $NF}')", 'env_path=$(sudo find $CI_PROJECT_DIR/environments -type d -name "$env_name")', 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done', 'cp -rf $CI_PROJECT_DIR/environments $CI_PROJECT_DIR/git_envs', - ], + ], } git_commit_vars = { @@ -111,9 +112,6 @@ def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, de "envgen_image": "$envgen_image", "envgen_args": " -vv", "envgen_debug": "true", - "module_ansible_dir": "/module/ansible", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_ansible_cfg": "/module/ansible/ansible.cfg", "module_config_default": "/module/templates/defaults.yaml", "GIT_STRATEGY": "none", "COMMIT_ENV": "true", @@ -128,4 +126,4 @@ def prepare_git_commit_job(pipeline, full_env, enviroment_name, cluster_name, de if (credential_rotation_job is not None): git_commit_job.add_needs(credential_rotation_job) pipeline.add_children(git_commit_job) - return git_commit_job \ No newline at end of file + return git_commit_job diff --git a/build_pipegene/scripts/gitlab_ci.py b/build_pipegene/scripts/gitlab_ci.py index df060fe64..bf25eab97 100644 --- a/build_pipegene/scripts/gitlab_ci.py +++ b/build_pipegene/scripts/gitlab_ci.py @@ -119,8 +119,8 @@ def build_pipeline(params: dict) -> None: cluster_name, tags) jobs_map["credential_rotation_job"] = credential_rotation_job else: - logger.info(f'Credential rotation job for {full_env_name} is skipped because CRED_ROTATION_PAYLOAD is empty.') - + logger.info( + f'Credential rotation job for {full_env_name} is skipped because CRED_ROTATION_PAYLOAD is empty.') if params['ENV_BUILD']: jobs_map["appregdef_render_job"] = prepare_appregdef_render_job(pipeline, params['IS_TEMPLATE_TEST'], @@ -131,8 +131,11 @@ def build_pipeline(params: dict) -> None: else: logger.info(f'Preparing of appregdef_render_job {full_env_name} is skipped.') - if (params["SD_SOURCE_TYPE"].lower() == "json" and params["SD_DATA"]) or \ - (params["SD_SOURCE_TYPE"].lower() == "artifact" and params["SD_VERSION"]): + source_type = (params.get("SD_SOURCE_TYPE", "artifact")).lower() + if ( + (source_type == "json" and params.get("SD_DATA")) or + (source_type == "artifact" and params.get("SD_VERSION")) + ): jobs_map["process_sd_job"] = prepare_process_sd(pipeline, full_env_name, environment_name, cluster_name, params["APP_DEFS_PATH"], params["REG_DEFS_PATH"], tags) else: diff --git a/build_pipegene/scripts/inventory_generation_job.py b/build_pipegene/scripts/inventory_generation_job.py index 8d59703ae..312cb91f9 100644 --- a/build_pipegene/scripts/inventory_generation_job.py +++ b/build_pipegene/scripts/inventory_generation_job.py @@ -47,9 +47,6 @@ def prepare_inventory_generation_job(pipeline, full_env_name, environment_name, "envgen_image": "$envgen_image", "envgen_args": " -vv", "envgen_debug": "true", - "module_ansible_dir": "/module/ansible", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", - "module_ansible_cfg": "/module/ansible/ansible.cfg", "module_config_default": "/module/templates/defaults.yaml", "GITLAB_RUNNER_TAG_NAME": tags, **env_generation_params diff --git a/build_pipegene/scripts/passport_jobs.py b/build_pipegene/scripts/passport_jobs.py index 0d6499eee..e7bc24d91 100644 --- a/build_pipegene/scripts/passport_jobs.py +++ b/build_pipegene/scripts/passport_jobs.py @@ -45,7 +45,7 @@ def prepare_passport_job(pipeline, full_env, enviroment_name, cluster_name, tags 'for path in $env_path; do if [ -d "$path/Credentials" ]; then sudo chmod ugo+rw $path/Credentials/*; fi; done' ], } - get_passport_params['script'].append('/module/scripts/prepare.sh "git_commit.yaml"') + get_passport_params['script'].append('/module/scripts/git_commit.sh') get_passport_vars = { "ENV_NAME": full_env, "CLUSTER_NAME": cluster_name, @@ -53,13 +53,10 @@ def prepare_passport_job(pipeline, full_env, enviroment_name, cluster_name, tags "envgen_image": "$envgen_image", "envgen_args": " -vv", "envgen_debug": "true", - "module_inventory": "${CI_PROJECT_DIR}/configuration/inventory.yaml", "module_config_default": "/module/templates/defaults.yaml", "COMMIT_ENV": "false", "COMMIT_MESSAGE": f"[ci_skip] update cloud passport for {cluster_name}", - "GITLAB_RUNNER_TAG_NAME": tags, - "module_ansible_dir": "/module/ansible", - "module_ansible_cfg": "/module/ansible/ansible.cfg" + "GITLAB_RUNNER_TAG_NAME": tags } get_passport_job = job_instance(params=get_passport_params, vars=get_passport_vars) base = "${CI_PROJECT_DIR}/environments" @@ -68,3 +65,4 @@ def prepare_passport_job(pipeline, full_env, enviroment_name, cluster_name, tags get_passport_job.artifacts.when = WhenStatement.ALWAYS pipeline.add_children(get_passport_job) return get_passport_job + diff --git a/dependencies/tests_requirements.txt b/dependencies/tests_requirements.txt index 73b663ee1..b37d7ad4a 100644 --- a/dependencies/tests_requirements.txt +++ b/dependencies/tests_requirements.txt @@ -23,8 +23,6 @@ referencing==0.33.0 rpds-py==0.17.1 jsonschema-specifications==2023.12.1 cryptography==41.0.3 -ansible-core==2.17.12 -ansible_runner==2.3.5 pytest==7.4.3 junitparser==3.1.2 hiyapyco==0.6.0 diff --git a/github_workflows/instance-repo-pipeline/.github/configuration/config.env b/github_workflows/instance-repo-pipeline/.github/configuration/config.env index 8bc9db08b..825824b2d 100644 --- a/github_workflows/instance-repo-pipeline/.github/configuration/config.env +++ b/github_workflows/instance-repo-pipeline/.github/configuration/config.env @@ -8,7 +8,4 @@ PROJECT_DIR=/workspace SECRET_POSTFIX=custom_secret envgen_args=-vvv envgen_debug=true -module_ansible_cfg=/module/ansible/ansible.cfg -module_ansible_dir=/module/ansible module_config_default=/module/templates/defaults.yaml -module_inventory=/workspace/configuration/inventory.yaml diff --git a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml index 2a3d0ae1d..05b5aaa00 100644 --- a/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml +++ b/gsf_packages/envgene_instance_project/git-system-follower-package/scripts/templates/default/{{ cookiecutter.gsf_repository_name }}/.gitlab-ci.yml @@ -32,9 +32,6 @@ default: .common_module.variables: variables: - module_ansible_dir: "/module/ansible" - module_inventory: "${CI_PROJECT_DIR}/configuration/inventory.yaml" - module_ansible_cfg: "/module/ansible/ansible.cfg" module_config_default: "/module/templates/defaults.yaml" .images.variables: diff --git a/scripts/utils/pipeline_parameters.py b/scripts/utils/pipeline_parameters.py index f0c43e2cb..6aa9d9ce0 100644 --- a/scripts/utils/pipeline_parameters.py +++ b/scripts/utils/pipeline_parameters.py @@ -15,7 +15,7 @@ def get_pipeline_parameters() -> dict: 'IS_TEMPLATE_TEST': getenv("ENV_TEMPLATE_TEST") == "true", 'CI_COMMIT_REF_NAME': getenv("CI_COMMIT_REF_NAME", ""), 'JSON_SCHEMAS_DIR': getenv("JSON_SCHEMAS_DIR", "/module/schemas"), - "SD_SOURCE_TYPE": getenv("SD_SOURCE_TYPE"), + "SD_SOURCE_TYPE": getenv("SD_SOURCE_TYPE") or "artifact", "SD_VERSION": getenv("SD_VERSION"), "SD_DATA": getenv("SD_DATA"), "SD_DELTA": getenv("SD_DELTA"),