feat: Add Python to Effective set image

build_effective_set_generator/.gitignore

@@ -36,6 +36,9 @@
 **/build.gradle
 **/settings.gradle
 **/build
+# Exception: build directory for Docker build files (must be after **/build rule)
+!build/
+!build/**
 
 # CMake
 cmake-build-debug/

build_effective_set_generator/Dockerfile

@@ -1,4 +1,7 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 AS builder
+#############################################
+# STAGE 1: CLI (UBI minimal runtime with Java)
+#############################################
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1 AS cli
 
 ARG JAVA_PACKAGE=java-17-openjdk-headless
 ARG RUN_JAVA_VERSION=1.3.8
@@ -7,28 +10,102 @@ ARG RUN_JAVA_VERSION=1.3.8
 RUN microdnf install -y curl ca-certificates ${JAVA_PACKAGE} && \
     microdnf clean all && \
     mkdir -p /deployments && \
-    curl -s https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh -o /deployments/run-java.sh && \
+    curl -sSf https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh \
+        -o /deployments/run-java.sh && \
     chmod 540 /deployments/run-java.sh && \
     echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/lib/security/java.security
 
 COPY build_effective_set_generator/effective-set-generator/target/*.jar /deployments/app.jar
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1
+
+
+#############################################
+# STAGE 2: Build Python Environment (Alpine)
+#############################################
+FROM python:3.12-alpine3.19 AS build
+
+# Install ALL system-level *build dependencies*
+# hadolint ignore=DL3018
+RUN apk add --no-cache \
+    gcc \
+    musl-dev \
+    libffi-dev \
+    openssl-dev \
+    python3-dev \
+    cargo \
+    rust \
+    build-base \
+    curl \
+    wget
+
+COPY build_effective_set_generator/build/configuration/pip.conf /etc/pip.conf
+COPY build_effective_set_generator/build/configuration/requirements.txt /build/requirements.txt
+COPY build_effective_set_generator/build/configuration/constraint.txt /build/constraint.txt
+
+# Build Python virtual environment
+RUN python3 -m venv /module/venv && \
+    /module/venv/bin/pip install --upgrade pip setuptools wheel && \
+    /module/venv/bin/pip install --no-cache-dir --no-binary cffi --no-binary cryptography \
+        -r /build/requirements.txt
+
+# Install SOPS
+RUN wget --quiet --tries=3 \
+    https://github.com/mozilla/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 \
+    -O /usr/local/bin/sops && \
+    chmod +x /usr/local/bin/sops
+
+# DO NOT delete build dependencies here — runtime needs them indirectly
+
+
+
+
+#############################################
+# STAGE 3: Final Runtime (Alpine)
+#############################################
+FROM python:3.12-alpine3.19 AS runtime
 
 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en'
 
-COPY --from=builder --chown=1001:root /deployments /deployments
-COPY --from=builder /etc/alternatives/jre/lib/security/java.security /etc/alternatives/jre/lib/security/java.security
+COPY --from=cli --chown=1001:root /deployments /deployments
+COPY --from=cli /etc/alternatives/jre/lib/security/java.security \
+     /etc/alternatives/jre/lib/security/java.security
+
+RUN chmod g+rwX /deployments
+
+COPY build_effective_set_generator/build/configuration/pip.conf /etc/pip.conf
+COPY build_effective_set_generator/build/configuration/constraint.txt /build/constraint.txt
+COPY build_effective_set_generator/build/configuration/sources.list /etc/apk/repositories
 
-## NOTE: This script requires Python and will fail unless Python is added in the future.
+# Install ONLY runtime dependencies (NO dev packages)
+# hadolint ignore=DL3018
+RUN apk add --no-cache \
+    libffi \
+    openssl \
+    bash \
+    ca-certificates \
+    tar \
+    curl \
+    jq \
+    yq \
+    gettext \
+    sed \
+    age
+
+# Bring the virtual environment compiled in build stage
+COPY --from=build /module/venv /module/venv
+COPY --from=build /usr/local/bin/sops /usr/local/bin/sops
+COPY build_effective_set_generator/build/scripts /module/scripts
 COPY scripts/utils /module/scripts/utils
 
-RUN chmod g+rwX /deployments
+# User creation + permissions
+RUN addgroup ci && adduser -D -h /module/ -s /bin/bash -G ci ci && \
+    chown ci:ci -R /module && \
+    chmod +x /module/scripts/*.sh && \
+    chmod 644 /module/scripts/*.py 2>/dev/null || true && \
+    chmod +x /usr/local/bin/sops
 
-# Ensure sane permissions on copied tree without findutils
-RUN chmod -R u=rwX,go=rX /deployments && \
-    chmod 540 /deployments/run-java.sh
+ENV PATH=/module/venv/bin:$PATH
 
-USER 1001
+USER ci:ci
 
-ENTRYPOINT [ "/deployments/run-java.sh" ]
+ENTRYPOINT [""]

build_effective_set_generator/build/configuration/constraint.txt

@@ -0,0 +1 @@
+cython<3

build_effective_set_generator/build/configuration/pip.conf

@@ -0,0 +1,5 @@
+[global]
+index-url=https://pypi.org/simple
+extra-index-url=https://example.com/pypi/simple
+trusted-host=pypi.org
+# constraint=/build/constraint.txt

build_effective_set_generator/build/configuration/requirements.txt

@@ -0,0 +1,9 @@
+# Base module requirements (essential only)
+shyaml==0.6.2
+yamale==4.0.4
+prettytable==3.5.0
+cryptography==38.0.0
+pyyaml>=6.0
+PyGithub==1.55
+certifi==2022.6.15
+GitPython==3.1.45

build_effective_set_generator/build/configuration/sources.list

@@ -0,0 +1,2 @@
+https://dl-cdn.alpinelinux.org/alpine/v3.20/main
+https://dl-cdn.alpinelinux.org/alpine/v3.20/community

build_effective_set_generator/build/scripts/decrypt.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+# Ensure externally provided variables are defined to satisfy shellcheck
+env_name="${env_name:-}"
+encrypt_file_path="${encrypt_file_path:-}"
+DECRYPT_TYPE="${DECRYPT_TYPE:-}"
+module_fernet_key_name="${module_fernet_key_name:-}"
+module_age_key_name="${module_age_key_name:-}"
+
+FERNET_KEY="CREDENTIALS_SECRET_KEY_${env_name}"
+SOPS_AGE_PRIVATE_KEY="AGE_SECRET_KEY_${env_name}"
+
+if [ -n "${module_fernet_key_name}" ]; then
+      FERNET_KEY="${module_fernet_key_name}"
+fi
+if [ -n "${module_age_key_name}" ]; then
+      SOPS_AGE_PRIVATE_KEY="${module_age_key_name}"
+fi
+
+if [ -n "${!FERNET_KEY}" ] && [ -f "${encrypt_file_path}" ] && [ "${DECRYPT_TYPE}" = 'fernet' ]; then
+
+      echo "${encrypt_file_path} exists and key variable is defined"
+      python /module/scripts/decrypt_fernet.py decrypt_cred_file --file_path "${encrypt_file_path}" --secret_key "${!FERNET_KEY}"
+elif [ -n "${!SOPS_AGE_PRIVATE_KEY}" ] && [ -f "${encrypt_file_path}" ] && [ "${DECRYPT_TYPE}" = 'sops' ]; then
+      SOPS_AGE_KEY="${!SOPS_AGE_PRIVATE_KEY}"
+      export SOPS_AGE_KEY
+      sops --decrypt -i "${encrypt_file_path}"
+      echo "${encrypt_file_path} was decrypted"
+elif [ "${DECRYPT_TYPE}" == 'none' ]; then
+      echo "Skipping decryption...."
+else
+      echo "Variable encrypt_file_path not exists or key variable is undefined. encrypt_file_path: '$encrypt_file_path'"
+fi

build_effective_set_generator/build/scripts/decrypt_fernet.py

@@ -0,0 +1,81 @@
+from envgenehelper import logger
+
+from yaml import safe_load, safe_dump
+import click
+from cryptography.fernet import Fernet
+
+ENCRYPTED_CONST = 'encrypted:AES256_Fernet'
+
+@click.group(chain=True)
+def cmdb_prepare():
+    pass
+
+
+@cmdb_prepare.command("decrypt_cred_file")
+@click.option('--file_path', '-f', 'file_path', required=True, help="Path to creds file")
+@click.option('--secret_key', '-s', 'secret_key', required=True,
+               help="Set secret_key for encrypt cred files")
+def decrypt_file(secret_key, file_path):
+    ''' {getenv('CI_PROJECT_DIR')}/ansible/inventory/group_vars/{getenv('env_name')}/appdeployer_cmdb/Tenants/{getenv('tenant_name')}/Credentials'''
+    logger.debug('Try to read %s file', file_path)
+    with open(file_path, mode="r", encoding="utf-8") as sensitive:
+        sensitive_data = safe_load(sensitive)
+
+    is_encrypted = check_if_file_is_encrypted(sensitive_data)
+    if is_encrypted:
+        if not secret_key:
+            logger.error(f'Variable "{secret_key}" is not specified')
+            exit(1)
+        cipher = Fernet(secret_key)
+        logger.debug('Try to decrypt data from %s file', file_path)
+        if isinstance(sensitive_data, dict):
+            decrypted_data = decode_sensitive(cipher, sensitive_data)
+            logger.debug('Try to write data to %s file', file_path)
+            with open(file_path, mode="w") as sensitive:
+                safe_dump(decrypted_data, sensitive, default_flow_style=False)
+                logger.info('The %s file has been decrypted', file_path)
+        else:
+            logger.info('The %s is empty or has no dict struct or not encrypted', file_path)
+    else:
+        logger.info('File is not encrypted')
+
+def check_if_file_is_encrypted(sensitive_data) -> bool:
+    for key, data in sensitive_data.items():
+        if key != "type" and key != "credentialsId" and data:
+            if isinstance(data, dict):
+                if check_if_file_is_encrypted(data):
+                    return True
+            elif isinstance(data, str):
+                if ENCRYPTED_CONST in data:
+                    return True
+            elif isinstance(data, list):
+                for item in data:
+                    if ENCRYPTED_CONST in item:
+                        return True
+
+    return False
+
+
+
+def decode_sensitive(cipher:Fernet, sensitive_data) -> str:
+    for key, data in sensitive_data.items():
+        if key != "type" and key != "credentialsId" and data:
+            if isinstance(data, dict):
+                decode_sensitive(cipher, data)
+            elif isinstance(data, list):
+                _list = []
+                for item in data:
+                    if ENCRYPTED_CONST in item:
+                        _list.append(
+                            cipher.decrypt(item.replace(
+                                f'[{ENCRYPTED_CONST}]','').encode('utf-8')).decode('utf-8'))
+                sensitive_data[key] = _list
+            elif ENCRYPTED_CONST in data:
+                sensitive_data[key] = cipher.decrypt(
+                    data.replace(f'[{ENCRYPTED_CONST}]','').encode('utf-8')).decode('utf-8')
+    return sensitive_data
+
+
+if __name__ == "__main__":
+    # yaml = create_yaml_processor()
+    cmdb_prepare()

build_effective_set_generator/build/scripts/get_include_list.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+CI_FILE="$1"
+
+include_list=$(shyaml get-value include <"${CI_FILE}" 2>/dev/null || true)
+if [[ "$include_list" != "" ]]; then
+      include_list_length=$(($(shyaml get-value include <"${CI_FILE}" | shyaml get-length) - 1))
+      for i in $(seq 0 $include_list_length); do
+          include_project=$(shyaml get-value include."$i" <"${CI_FILE}" 2>/dev/null | shyaml get-value project 2>/dev/null || true)
+          if [[ "$include_project" != "" ]]; then
+              include_project_full_path=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value project)
+              include_project_branch=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value ref)
+              include_project_file=$(shyaml get-value include."$i" <"${CI_FILE}" | shyaml get-value file)
+
+              include_project_group=${include_project_full_path%/*}
+              include_project_repo=${include_project_full_path##*/}
+
+              if [[ "$include_project_file" == *"api.yaml"* || "$include_project_file" == *"pipeline.yaml"* ]]; then
+                  module_project_path_result=$(env | grep "${include_project_full_path}") || true
+                  IFS='=' read -r -a module_project_path_array <<<"$module_project_path_result"
+                  module_project_path=${module_project_path_array[0]}
+
+                  module_project=${include_project_repo}
+                  module_group=${include_project_group}
+                  module_full_path=${include_project_full_path}
+                  module_version=${include_project_branch}
+                  module_name=${module_project_path//_project_path/}
+
+                  # if <module_name>_project_path not specifed
+                  : "${module_name:=$module_project}"
+
+                  cat <<YAML
+- name: ${module_name}
+      project: ${module_project}
+      group: ${module_group}
+      full_path: ${module_full_path}
+      version: ${module_version}
+YAML
+              else
+                  echo "Included file is not a module or pipeline template ($include_project_file)" >/dev/stderr
+              fi
+          else
+              unsupported_type=$(shyaml get-value include."$i" <"${CI_FILE}" 2>/dev/null || true)
+              echo "Included file of unsupported type (${unsupported_type}). Only inlude:file is supported now" >/dev/stderr
+          fi
+      done
+else
+      echo "Nothing to include. Check that your ci file contains include section" >/dev/stderr
+fi

build_effective_set_generator/build/scripts/show_validate.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import yaml
+import glob
+import argparse
+from prettytable import PrettyTable, ALL
+
+parser = argparse.ArgumentParser(description="Get variables from repository")
+parser.add_argument("-p", "--path", dest="dest_dir", type=str, help='Path to folder validation')
+parser.add_argument("-n", "--name", dest="place_validation", type=str, help='Module name')
+args = parser.parse_args()
+
+
+header = ['Module name', 'Place of validation', 'Validation status']
+table = PrettyTable(header, align='l')
+table._max_width = {'Module name' : 20, 'Place of validation' : 20, 'Validation status' : 90}
+table.hrules = ALL
+yaml_file = []
+if args.place_validation:
+    with open(f"{args.dest_dir}/{args.place_validation}_validation.yaml", "r") as report:
+    #with open(f"dvm_validation.yaml", "r") as file:
+        yaml_file = yaml.safe_load(report)
+    for module in yaml_file:
+        table.add_row([module['module_name'], module['place_of_validation'], module['validation_status']])
+else:
+    file_list = glob.glob(f"{args.dest_dir}/*_validation.yaml")
+    for file in file_list:
+        with open(file, "r") as report:
+            yaml_content = yaml.safe_load(report)
+            for module in yaml_content:
+                yaml_file.append(module)
+    for module in yaml_file:
+        if len(module['validation_status'].split("\n"))>1:
+            messages = ''
+            for error in module['validation_status'].split("\n")[1:]:
+                if error:
+                    messages = messages + "\n\033[33m"+error.split(':')[0]+":\033[39m"+":".join(error.split(':')[1:])
+            module['validation_status'] = module['validation_status'].split("\n")[0] + messages
+        table.add_row([module['module_name'], module['place_of_validation'], module['validation_status']])
+print(table)
+