diff --git a/build_effective_set_generator/.gitignore b/build_effective_set_generator/.gitignore index a572cf5b7..25ff83d14 100644 --- a/build_effective_set_generator/.gitignore +++ b/build_effective_set_generator/.gitignore @@ -36,6 +36,9 @@ **/build.gradle **/settings.gradle **/build +# Exception: build directory for Docker build files (must be after **/build rule) +!build/ +!build/** # CMake cmake-build-debug/ diff --git a/build_effective_set_generator/Dockerfile b/build_effective_set_generator/Dockerfile index e9cd485cd..ca57f17b4 100644 --- a/build_effective_set_generator/Dockerfile +++ b/build_effective_set_generator/Dockerfile @@ -1,4 +1,7 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 AS builder +############################################# +# STAGE 1: CLI (UBI minimal runtime with Java) +############################################# +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 AS cli ARG JAVA_PACKAGE=java-17-openjdk-headless ARG RUN_JAVA_VERSION=1.3.8 @@ -7,28 +10,102 @@ ARG RUN_JAVA_VERSION=1.3.8 RUN microdnf install -y curl ca-certificates ${JAVA_PACKAGE} && \ microdnf clean all && \ mkdir -p /deployments && \ - curl -s https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh -o /deployments/run-java.sh && \ + curl -sSf https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh \ + -o /deployments/run-java.sh && \ chmod 540 /deployments/run-java.sh && \ echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/lib/security/java.security COPY build_effective_set_generator/effective-set-generator/target/*.jar /deployments/app.jar -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 + + +############################################# +# STAGE 2: Build Python Environment (Alpine) +############################################# +FROM python:3.12-alpine3.19 AS build + +# Install ALL system-level *build dependencies* +# hadolint ignore=DL3018 +RUN apk add --no-cache \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python3-dev \ + cargo \ + rust \ + build-base \ + curl \ + wget + +COPY build_effective_set_generator/build/configuration/pip.conf /etc/pip.conf +COPY build_effective_set_generator/build/configuration/requirements.txt /build/requirements.txt +COPY build_effective_set_generator/build/configuration/constraint.txt /build/constraint.txt + +# Build Python virtual environment +RUN python3 -m venv /module/venv && \ + /module/venv/bin/pip install --upgrade pip setuptools wheel && \ + /module/venv/bin/pip install --no-cache-dir --no-binary cffi --no-binary cryptography \ + -r /build/requirements.txt + +# Install SOPS +RUN wget --quiet --tries=3 \ + https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \ + -O /usr/local/bin/sops && \ + chmod +x /usr/local/bin/sops + +# DO NOT delete build dependencies here — runtime needs them indirectly + + + + +############################################# +# STAGE 3: Final Runtime (Alpine) +############################################# +FROM python:3.12-alpine3.19 AS runtime ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' -COPY --from=builder --chown=1001:root /deployments /deployments -COPY --from=builder /etc/alternatives/jre/lib/security/java.security /etc/alternatives/jre/lib/security/java.security +COPY --from=cli --chown=1001:root /deployments /deployments +COPY --from=cli /etc/alternatives/jre/lib/security/java.security \ + /etc/alternatives/jre/lib/security/java.security + +RUN chmod g+rwX /deployments + +COPY build_effective_set_generator/build/configuration/pip.conf /etc/pip.conf +COPY build_effective_set_generator/build/configuration/constraint.txt /build/constraint.txt +COPY build_effective_set_generator/build/configuration/sources.list /etc/apk/repositories -## NOTE: This script requires Python and will fail unless Python is added in the future. +# Install ONLY runtime dependencies (NO dev packages) +# hadolint ignore=DL3018 +RUN apk add --no-cache \ + libffi \ + openssl \ + bash \ + ca-certificates \ + tar \ + curl \ + jq \ + yq \ + gettext \ + sed \ + age + +# Bring the virtual environment compiled in build stage +COPY --from=build /module/venv /module/venv +COPY --from=build /usr/local/bin/sops /usr/local/bin/sops +COPY build_effective_set_generator/build/scripts /module/scripts COPY scripts/utils /module/scripts/utils -RUN chmod g+rwX /deployments +# User creation + permissions +RUN addgroup ci && adduser -D -h /module/ -s /bin/bash -G ci ci && \ + chown ci:ci -R /module && \ + chmod +x /module/scripts/*.sh && \ + chmod 644 /module/scripts/*.py 2>/dev/null || true && \ + chmod +x /usr/local/bin/sops -# Ensure sane permissions on copied tree without findutils -RUN chmod -R u=rwX,go=rX /deployments && \ - chmod 540 /deployments/run-java.sh +ENV PATH=/module/venv/bin:$PATH -USER 1001 +USER ci:ci -ENTRYPOINT [ "/deployments/run-java.sh" ] +ENTRYPOINT [""] diff --git a/build_effective_set_generator/build/configuration/constraint.txt b/build_effective_set_generator/build/configuration/constraint.txt new file mode 100644 index 000000000..039eb0db2 --- /dev/null +++ b/build_effective_set_generator/build/configuration/constraint.txt @@ -0,0 +1 @@ +cython<3 diff --git a/build_effective_set_generator/build/configuration/pip.conf b/build_effective_set_generator/build/configuration/pip.conf new file mode 100644 index 000000000..31058a45f --- /dev/null +++ b/build_effective_set_generator/build/configuration/pip.conf @@ -0,0 +1,5 @@ +[global] +index-url=https://pypi.org/simple +extra-index-url=https://example.com/pypi/simple +trusted-host=pypi.org +# constraint=/build/constraint.txt diff --git a/build_effective_set_generator/build/configuration/requirements.txt b/build_effective_set_generator/build/configuration/requirements.txt new file mode 100644 index 000000000..3a4cb8dfa --- /dev/null +++ b/build_effective_set_generator/build/configuration/requirements.txt @@ -0,0 +1,9 @@ +# Base module requirements (essential only) +shyaml==0.6.2 +yamale==4.0.4 +prettytable==3.5.0 +cryptography==38.0.0 +pyyaml>=6.0 +PyGithub==1.55 +certifi==2022.6.15 +GitPython==3.1.45 diff --git a/build_effective_set_generator/build/configuration/sources.list b/build_effective_set_generator/build/configuration/sources.list new file mode 100644 index 000000000..91b969a08 --- /dev/null +++ b/build_effective_set_generator/build/configuration/sources.list @@ -0,0 +1,2 @@ +https://dl-cdn.alpinelinux.org/alpine/v3.20/main +https://dl-cdn.alpinelinux.org/alpine/v3.20/community diff --git a/build_effective_set_generator/build/scripts/decrypt.sh b/build_effective_set_generator/build/scripts/decrypt.sh new file mode 100755 index 000000000..715667598 --- /dev/null +++ b/build_effective_set_generator/build/scripts/decrypt.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# Ensure externally provided variables are defined to satisfy shellcheck +env_name="${env_name:-}" +encrypt_file_path="${encrypt_file_path:-}" +DECRYPT_TYPE="${DECRYPT_TYPE:-}" +module_fernet_key_name="${module_fernet_key_name:-}" +module_age_key_name="${module_age_key_name:-}" + +FERNET_KEY="CREDENTIALS_SECRET_KEY_${env_name}" +SOPS_AGE_PRIVATE_KEY="AGE_SECRET_KEY_${env_name}" + +if [ -n "${module_fernet_key_name}" ]; then + FERNET_KEY="${module_fernet_key_name}" +fi +if [ -n "${module_age_key_name}" ]; then + SOPS_AGE_PRIVATE_KEY="${module_age_key_name}" +fi + +if [ -n "${!FERNET_KEY}" ] && [ -f "${encrypt_file_path}" ] && [ "${DECRYPT_TYPE}" = 'fernet' ]; then + + echo "${encrypt_file_path} exists and key variable is defined" + python /module/scripts/decrypt_fernet.py decrypt_cred_file --file_path "${encrypt_file_path}" --secret_key "${!FERNET_KEY}" +elif [ -n "${!SOPS_AGE_PRIVATE_KEY}" ] && [ -f "${encrypt_file_path}" ] && [ "${DECRYPT_TYPE}" = 'sops' ]; then + SOPS_AGE_KEY="${!SOPS_AGE_PRIVATE_KEY}" + export SOPS_AGE_KEY + sops --decrypt -i "${encrypt_file_path}" + echo "${encrypt_file_path} was decrypted" +elif [ "${DECRYPT_TYPE}" == 'none' ]; then + echo "Skipping decryption...." +else + echo "Variable encrypt_file_path not exists or key variable is undefined. encrypt_file_path: '$encrypt_file_path'" +fi diff --git a/build_effective_set_generator/build/scripts/decrypt_fernet.py b/build_effective_set_generator/build/scripts/decrypt_fernet.py new file mode 100644 index 000000000..86aa61192 --- /dev/null +++ b/build_effective_set_generator/build/scripts/decrypt_fernet.py @@ -0,0 +1,81 @@ +from envgenehelper import logger + +from yaml import safe_load, safe_dump +import click +from cryptography.fernet import Fernet + +ENCRYPTED_CONST = 'encrypted:AES256_Fernet' + +@click.group(chain=True) +def cmdb_prepare(): + pass + + +@cmdb_prepare.command("decrypt_cred_file") +@click.option('--file_path', '-f', 'file_path', required=True, help="Path to creds file") +@click.option('--secret_key', '-s', 'secret_key', required=True, + help="Set secret_key for encrypt cred files") +def decrypt_file(secret_key, file_path): + ''' {getenv('CI_PROJECT_DIR')}/ansible/inventory/group_vars/{getenv('env_name')}/appdeployer_cmdb/Tenants/{getenv('tenant_name')}/Credentials''' + logger.debug('Try to read %s file', file_path) + with open(file_path, mode="r", encoding="utf-8") as sensitive: + sensitive_data = safe_load(sensitive) + + is_encrypted = check_if_file_is_encrypted(sensitive_data) + if is_encrypted: + if not secret_key: + logger.error(f'Variable "{secret_key}" is not specified') + exit(1) + cipher = Fernet(secret_key) + logger.debug('Try to decrypt data from %s file', file_path) + if isinstance(sensitive_data, dict): + decrypted_data = decode_sensitive(cipher, sensitive_data) + logger.debug('Try to write data to %s file', file_path) + with open(file_path, mode="w") as sensitive: + safe_dump(decrypted_data, sensitive, default_flow_style=False) + logger.info('The %s file has been decrypted', file_path) + else: + logger.info('The %s is empty or has no dict struct or not encrypted', file_path) + else: + logger.info('File is not encrypted') + +def check_if_file_is_encrypted(sensitive_data) -> bool: + for key, data in sensitive_data.items(): + if key != "type" and key != "credentialsId" and data: + if isinstance(data, dict): + if check_if_file_is_encrypted(data): + return True + elif isinstance(data, str): + if ENCRYPTED_CONST in data: + return True + elif isinstance(data, list): + for item in data: + if ENCRYPTED_CONST in item: + return True + + return False + + + +def decode_sensitive(cipher:Fernet, sensitive_data) -> str: + for key, data in sensitive_data.items(): + if key != "type" and key != "credentialsId" and data: + if isinstance(data, dict): + decode_sensitive(cipher, data) + elif isinstance(data, list): + _list = [] + for item in data: + if ENCRYPTED_CONST in item: + _list.append( + cipher.decrypt(item.replace( + f'[{ENCRYPTED_CONST}]','').encode('utf-8')).decode('utf-8')) + sensitive_data[key] = _list + elif ENCRYPTED_CONST in data: + sensitive_data[key] = cipher.decrypt( + data.replace(f'[{ENCRYPTED_CONST}]','').encode('utf-8')).decode('utf-8') + return sensitive_data + + +if __name__ == "__main__": + # yaml = create_yaml_processor() + cmdb_prepare() diff --git a/build_effective_set_generator/build/scripts/get_include_list.sh b/build_effective_set_generator/build/scripts/get_include_list.sh new file mode 100755 index 000000000..68d6b07da --- /dev/null +++ b/build_effective_set_generator/build/scripts/get_include_list.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +CI_FILE="$1" + +include_list=$(shyaml get-value include <"${CI_FILE}" 2>/dev/null || true) +if [[ "$include_list" != "" ]]; then + include_list_length=$(($(shyaml get-value include <"${CI_FILE}" | shyaml get-length) - 1)) + for i in $(seq 0 $include_list_length); do + include_project=$(shyaml get-value include."$i" <"${CI_FILE}" 2>/dev/null | shyaml get-value project 2>/dev/null || true) + if [[ "$include_project" != "" ]]; then + include_project_full_path=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value project) + include_project_branch=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value ref) + include_project_file=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value file) + + include_project_group=${include_project_full_path%/*} + include_project_repo=${include_project_full_path##*/} + + if [[ "$include_project_file" == *"api.yaml"* || "$include_project_file" == *"pipeline.yaml"* ]]; then + module_project_path_result=$(env | grep "${include_project_full_path}") || true + IFS='=' read -r -a module_project_path_array <<<"$module_project_path_result" + module_project_path=${module_project_path_array[0]} + + module_project=${include_project_repo} + module_group=${include_project_group} + module_full_path=${include_project_full_path} + module_version=${include_project_branch} + module_name=${module_project_path//_project_path/} + + # if _project_path not specifed + : "${module_name:=$module_project}" + + cat </dev/stderr + fi + else + unsupported_type=$(shyaml get-value include."$i" <"${CI_FILE}" 2>/dev/null || true) + echo "Included file of unsupported type (${unsupported_type}). Only inlude:file is supported now" >/dev/stderr + fi + done +else + echo "Nothing to include. Check that your ci file contains include section" >/dev/stderr +fi diff --git a/build_effective_set_generator/build/scripts/show_validate.py b/build_effective_set_generator/build/scripts/show_validate.py new file mode 100644 index 000000000..fb90f32bb --- /dev/null +++ b/build_effective_set_generator/build/scripts/show_validate.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import yaml +import glob +import argparse +from prettytable import PrettyTable, ALL + +parser = argparse.ArgumentParser(description="Get variables from repository") +parser.add_argument("-p", "--path", dest="dest_dir", type=str, help='Path to folder validation') +parser.add_argument("-n", "--name", dest="place_validation", type=str, help='Module name') +args = parser.parse_args() + + +header = ['Module name', 'Place of validation', 'Validation status'] +table = PrettyTable(header, align='l') +table._max_width = {'Module name' : 20, 'Place of validation' : 20, 'Validation status' : 90} +table.hrules = ALL +yaml_file = [] +if args.place_validation: + with open(f"{args.dest_dir}/{args.place_validation}_validation.yaml", "r") as report: + #with open(f"dvm_validation.yaml", "r") as file: + yaml_file = yaml.safe_load(report) + for module in yaml_file: + table.add_row([module['module_name'], module['place_of_validation'], module['validation_status']]) +else: + file_list = glob.glob(f"{args.dest_dir}/*_validation.yaml") + for file in file_list: + with open(file, "r") as report: + yaml_content = yaml.safe_load(report) + for module in yaml_content: + yaml_file.append(module) + for module in yaml_file: + if len(module['validation_status'].split("\n"))>1: + messages = '' + for error in module['validation_status'].split("\n")[1:]: + if error: + messages = messages + "\n\033[33m"+error.split(':')[0]+":\033[39m"+":".join(error.split(':')[1:]) + module['validation_status'] = module['validation_status'].split("\n")[0] + messages + table.add_row([module['module_name'], module['place_of_validation'], module['validation_status']]) +print(table) + diff --git a/build_effective_set_generator/build/scripts/update_ca_certs.sh b/build_effective_set_generator/build/scripts/update_ca_certs.sh new file mode 100755 index 000000000..e6e42d240 --- /dev/null +++ b/build_effective_set_generator/build/scripts/update_ca_certs.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +CA_FILE="$1" + +function getLinuxDisto { + if [[ -f /etc/os-release ]]; then + # freedesktop.org and systemd + # shellcheck disable=SC1091 + . /etc/os-release + DIST=$NAME + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + DIST=$(lsb_release -si) + elif [[ -f /etc/lsb-release ]]; then + # For some versions of Debian/Ubuntu without lsb_release command + # shellcheck disable=SC1091 + . /etc/lsb-release + DIST=$DISTRIB_ID + elif [[ -f /etc/debian_version ]]; then + # Older Debian/Ubuntu/etc. + DIST=Debian + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + DIST=$(uname -s) + fi + # convert to lowercase + DIST="$(tr '[:upper:]' '[:lower:]' <<<"$DIST")" +} + +function updateCertificates { + if [[ -e "${CA_FILE}" && -n "${CA_FILE}" ]]; then + getLinuxDisto + echo "Linux Distribution identified as: $DIST" + if [[ "${DIST}" == *"debian"* || "${DIST}" == *"ubuntu"* ]]; then + cp "${CA_FILE}" /usr/local/share/ca-certificates/ca.crt + update-ca-certificates --fresh >/dev/null + elif [[ "${DIST}" == *"centos"* ]]; then + cp "${CA_FILE}" /etc/pki/ca-trust/source/anchors/ca.crt + update-ca-trust + elif [[ "${DIST}" == *"alpine"* ]]; then + cat "${CA_FILE}" >>/etc/ssl/certs/ca-certificates.crt + echo "certs from $CA_FILE added to trusted root" + fi + else + echo "CA file ${CA_FILE} not found or empty" + exit 1 + fi +} + +updateCertificates diff --git a/build_envgene/build/Dockerfile b/build_envgene/build/Dockerfile index 15dcbbec3..3e6390759 100644 --- a/build_envgene/build/Dockerfile +++ b/build_envgene/build/Dockerfile @@ -47,21 +47,21 @@ RUN python -m venv /module/venv RUN /module/venv/bin/pip install --upgrade pip "setuptools<82" wheel RUN /module/venv/bin/pip install --no-cache-dir --retries 10 --timeout 60 -r /build/requirements.txt -RUN /module/venv/bin/pip install /python/jschon-sort -RUN /module/venv/bin/pip install /python/envgene -RUN /module/venv/bin/pip install /python/integration -RUN /module/venv/bin/pip install /python/artifact-searcher -RUN /module/venv/bin/pip install --no-cache-dir --no-deps -r /build/creds_rotation_requirements.txt +RUN /module/venv/bin/pip install /python/jschon-sort && \ + /module/venv/bin/pip install /python/envgene && \ + /module/venv/bin/pip install /python/integration && \ + /module/venv/bin/pip install /python/artifact-searcher && \ + /module/venv/bin/pip install --no-cache-dir --no-deps -r /build/creds_rotation_requirements.txt # Download and install SOPS for secrets management -RUN wget --tries=3 \ +RUN wget --quiet --tries=3 \ https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \ -O /usr/local/bin/sops && \ chmod +x /usr/local/bin/sops # Aggressive cleanup to reduce image size -RUN apk del gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev zlib-dev -RUN rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache +RUN apk del gcc musl-dev libffi-dev openssl-dev libxml2-dev libxslt-dev zlib-dev && \ + rm -rf /var/cache/apk/* /tmp/* /var/tmp/* /root/.cache # Remove unnecessary files from Python packages RUN find /module/venv/lib/python3.12/site-packages -name '*.pyc' -delete @@ -72,8 +72,8 @@ RUN rm -rf /module/venv/lib/python3.12/site-packages/pytest* \ RUN /module/venv/bin/pip cache purge # Set permissions -RUN chmod 754 /module/scripts/* -RUN chmod 754 /module/creds_rotation_scripts/* +RUN chmod 754 /module/scripts/* && \ + chmod 754 /module/creds_rotation_scripts/* ######################################### # Stage 2: Runtime diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 194abd016..000000000 --- a/pom.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - 4.0.0 - - org.qubership - qubership_envgene_templates - ${revision} - - ${project.groupId}:${project.artifactId} - >A Maven artifact that contains templates for Qubership Envgene Instance - https://github.com/Netcracker/qubership-envgene - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0 - - - - - Netcracker - opensourcegroup@netcracker.com - Netcracker Technology - https://www.netcracker.com - - - - scm:git:git://github.com/Netcracker/qubership-envgene.git - scm:git:ssh://github.com:Netcracker/qubership-envgene.git - https://github.com/Netcracker/qubership-envgene/tree/main - - - - - central - Central Maven Repository - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.2.7 - - - sign-artifacts - verify - - sign - - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.3.0 - - - attach-sources - - jar-no-fork - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.6.3 - - - attach-javadocs - - jar - - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.6.0 - true - - central - true - published - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.6.0 - - - make-assembly - package - - single - - - - - - src/assembly/assembly.xml - - false - - - - - - - 0.0.0.1 - - diff --git a/src/assembly/assembly.xml b/src/assembly/assembly.xml deleted file mode 100644 index 6ab61e251..000000000 --- a/src/assembly/assembly.xml +++ /dev/null @@ -1,18 +0,0 @@ - - templates - - zip - - false - - - build_envgene_templates - / - - **/* - - - -