From 1171758e5dcc0c382ead9ce9057f7acb3aecdcbf Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 10:58:55 -0400 Subject: [PATCH 1/6] feat: jfrog integration --- src/commands/fetch-shipyard-env.yml | 9 +- src/commands/upload-to-jfrog.yml | 61 ++++++ src/scripts/jfrog_upload.py | 295 ++++++++++++++++++++++++++++ src/scripts/orb.py | 16 ++ src/scripts/orb.sh | 90 --------- src/scripts/requirements.txt | 3 + src/scripts/setup-python.sh | 67 +++++++ src/scripts/setup-shell.sh | 34 ++++ 8 files changed, 484 insertions(+), 91 deletions(-) create mode 100644 src/commands/upload-to-jfrog.yml create mode 100644 src/scripts/jfrog_upload.py delete mode 100644 src/scripts/orb.sh create mode 100644 src/scripts/setup-python.sh create mode 100755 src/scripts/setup-shell.sh diff --git a/src/commands/fetch-shipyard-env.yml b/src/commands/fetch-shipyard-env.yml index c3ed4cb..248aba2 100644 --- a/src/commands/fetch-shipyard-env.yml +++ b/src/commands/fetch-shipyard-env.yml @@ -33,9 +33,16 @@ steps: command: "[[ -z $CIRCLE_PULL_REQUEST ]] && circleci-agent task halt || echo This job is for: $CIRCLE_PULL_REQUEST" - jq/install + - run: + name: "Setup Python environment for Shipyard" + command: <> + - run: + name: "Setup shell" + command: <> - run: name: "Fetch the Shipyard environment variables" environment: SHIPYARD_TIMEOUT: <> SHIPYARD_APP_NAME: <> - command: <> + command: | + python orb.py diff --git a/src/commands/upload-to-jfrog.yml b/src/commands/upload-to-jfrog.yml new file mode 100644 index 0000000..879d33b --- /dev/null +++ b/src/commands/upload-to-jfrog.yml @@ -0,0 +1,61 @@ +description: > + Upload artifacts to JFrog with build evidence including URLs and build ID + +parameters: + + artifactory-url: + type: env_var_name + default: "ARTIFACTORY_URL" + description: "Environment variable containing JFrog Artifactory URL (standard JFrog orb variable)" + + artifactory-user: + type: env_var_name + default: "ARTIFACTORY_USER" + description: "Environment variable containing JFrog username (standard JFrog orb variable)" + + artifactory-apikey: + type: env_var_name + default: "ARTIFACTORY_APIKEY" + description: "Environment variable containing JFrog API key (standard JFrog orb variable)" + + repository: + type: string + description: "Target repository in JFrog Artifactory" + + build-name: + type: string + default: "${CIRCLE_PROJECT_REPONAME}" + description: "Build name for artifact evidence" + + build-number: + type: string + default: "${CIRCLE_BUILD_NUM}" + description: "Build number for artifact evidence" + + target-path: + type: string + default: "shipyard/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}/environment-data.json" + description: "Target path in JFrog repository for Shipyard environment data" + + environment-url: + type: string + default: "${SHIPYARD_ENVIRONMENT_URL}" + description: "Shipyard environment URL to include in build evidence" + +steps: + - run: + name: "Setup shell" + command: <> + - run: + name: "Upload artifact to JFrog with build evidence" + environment: + ARTIFACTORY_URL_VAR: <> + ARTIFACTORY_USER_VAR: <> + ARTIFACTORY_APIKEY_VAR: <> + ARTIFACTORY_REPOSITORY: <> + BUILD_NAME: <> + BUILD_NUMBER: <> + TARGET_PATH: <> + ENVIRONMENT_URL: <> + command: | + python jfrog_upload.py \ No newline at end of file diff --git a/src/scripts/jfrog_upload.py b/src/scripts/jfrog_upload.py new file mode 100644 index 0000000..196489a --- /dev/null +++ b/src/scripts/jfrog_upload.py @@ -0,0 +1,295 @@ +""" +# Purpose: + +Upload artifacts to JFrog Artifactory with complete Shipyard environment data +as build evidence including URLs, environment ID, and timestamps. + +# Required environment variables: + +- ARTIFACTORY_URL_VAR (name of env var containing Artifactory URL) +- ARTIFACTORY_USER_VAR (name of env var containing Artifactory username) +- ARTIFACTORY_APIKEY_VAR (name of env var containing Artifactory API key) +- ARTIFACTORY_REPOSITORY (target repository name) +- ARTIFACT_PATH (local path to artifact file) +- TARGET_PATH (target path in repository) +- BUILD_NAME (build name for evidence) +- BUILD_NUMBER (build number for evidence) +""" +from __future__ import print_function + +import os +import sys +import json +import hashlib +import requests +from datetime import datetime + + +def exit_with_error(msg): + print(f"❌ ERROR: {msg}") + sys.exit(1) + + +def calculate_checksums(file_path): + """Calculate MD5 and SHA1 checksums for the artifact file""" + md5_hash = hashlib.md5() + sha1_hash = hashlib.sha1() + + try: + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + md5_hash.update(chunk) + sha1_hash.update(chunk) + + return { + 'md5': md5_hash.hexdigest(), + 'sha1': sha1_hash.hexdigest() + } + except Exception as e: + exit_with_error(f"Failed to calculate checksums: {e}") + + +def read_shipyard_data(): + """Read complete Shipyard environment data from JSON file""" + json_file_path = '/tmp/shipyard_environment_data.json' + + if not os.path.exists(json_file_path): + print("⚠️ No Shipyard environment JSON file found - using basic environment variables only") + return None + + try: + with open(json_file_path, 'r') as f: + data = json.load(f) + + print("📊 Reading Shipyard environment data from JSON file") + print(f"Environment ID: {data.get('environment_id', 'N/A')}") + print(f"Main URL: {data.get('environment_data', {}).get('url', 'N/A')}") + print(f"Fetched at: {data.get('fetched_at', 'N/A')}") + + return data + except Exception as e: + print(f"⚠️ Failed to read Shipyard JSON data: {e}") + return None + + +def create_build_properties(shipyard_data): + """Create build properties including Shipyard environment data""" + properties = { + 'circle.build.number': os.environ.get('CIRCLE_BUILD_NUM', ''), + 'circle.job': os.environ.get('CIRCLE_JOB', ''), + 'circle.workflow.id': os.environ.get('CIRCLE_WORKFLOW_ID', ''), + 'circle.pr.number': os.environ.get('CIRCLE_PR_NUMBER', ''), + } + + if shipyard_data: + env_data = shipyard_data.get('environment_data', {}) + properties.update({ + 'shipyard.environment.id': shipyard_data.get('environment_id', ''), + 'shipyard.environment.url': env_data.get('url', ''), + 'shipyard.environment.ready': str(env_data.get('ready', '')), + 'shipyard.environment.retired': str(env_data.get('retired', '')), + 'shipyard.environment.fetched_at': shipyard_data.get('fetched_at', ''), + 'shipyard.environment.commit_hash': shipyard_data.get('commit_hash', ''), + }) + + # Add additional URLs as separate properties + additional_urls = shipyard_data.get('additional_urls', {}) + for key, value in additional_urls.items(): + properties[f'shipyard.url.{key}'] = value + + # Add projects data as JSON string + projects = env_data.get('projects', []) + if projects: + properties['shipyard.projects.data'] = json.dumps(projects) + + return properties + + +def upload_artifact(artifactory_url, artifactory_user, artifactory_apikey, + repository, artifact_path, target_path, checksums): + """Upload artifact to JFrog Artifactory""" + + full_url = f"{artifactory_url}/{repository}/{target_path}" + + print("📤 Uploading artifact to JFrog...") + print(f"Source: {artifact_path}") + print(f"Target: {full_url}") + + headers = { + 'X-JFrog-Art-Api': artifactory_apikey, + 'X-Checksum-Deploy': 'false', + 'X-Checksum-Sha1': checksums['sha1'], + 'X-Checksum-Md5': checksums['md5'] + } + + try: + with open(artifact_path, 'rb') as f: + response = requests.put( + full_url, + auth=(artifactory_user, artifactory_apikey), + headers=headers, + data=f, + timeout=300 + ) + + if response.status_code in [200, 201]: + print("✅ Artifact uploaded successfully!") + return True + else: + print(f"❌ Upload failed with HTTP code: {response.status_code}") + print(f"Response: {response.text}") + return False + + except Exception as e: + exit_with_error(f"Failed to upload artifact: {e}") + + +def upload_build_info(artifactory_url, artifactory_user, artifactory_apikey, + build_name, build_number, target_path, checksums, properties): + """Upload build information to JFrog Artifactory""" + + build_info = { + "version": "1.0.1", + "name": build_name, + "number": build_number, + "type": "GENERIC", + "buildAgent": { + "name": "CircleCI", + "version": "2.1" + }, + "agent": { + "name": "shipyard-orb", + "version": "1.0.0" + }, + "started": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ')[:-3] + 'Z', + "durationMillis": 0, + "principal": os.environ.get('CIRCLE_USERNAME', 'circleci'), + "artifactoryPrincipal": artifactory_user, + "url": os.environ.get('CIRCLE_BUILD_URL', ''), + "vcs": { + "revision": os.environ.get('CIRCLE_SHA1', ''), + "url": os.environ.get('CIRCLE_REPOSITORY_URL', '') + }, + "modules": [ + { + "id": build_name, + "artifacts": [ + { + "type": target_path.split('.')[-1] if '.' in target_path else 'generic', + "sha1": checksums['sha1'], + "md5": checksums['md5'], + "name": os.path.basename(target_path), + "path": target_path + } + ] + } + ], + "properties": properties + } + + build_info_url = f"{artifactory_url}/api/build" + + print("📋 Uploading build evidence...") + + try: + response = requests.put( + build_info_url, + auth=(artifactory_user, artifactory_apikey), + headers={'Content-Type': 'application/json'}, + json=build_info, + timeout=60 + ) + + if response.status_code in [200, 204]: + print("✅ Build evidence uploaded successfully!") + print(f"Build Name: {build_name}") + print(f"Build Number: {build_number}") + return True + else: + print(f"⚠️ Build evidence upload failed with HTTP code: {response.status_code}") + print(f"Response: {response.text}") + print("Artifact was uploaded but build evidence failed") + return False + + except Exception as e: + print(f"⚠️ Failed to upload build evidence: {e}") + print("Artifact was uploaded but build evidence failed") + return False + + +def main(): + """Main function to orchestrate the JFrog upload process""" + + # Get credentials from environment variables + artifactory_url_var = os.environ.get('ARTIFACTORY_URL_VAR') + artifactory_user_var = os.environ.get('ARTIFACTORY_USER_VAR') + artifactory_apikey_var = os.environ.get('ARTIFACTORY_APIKEY_VAR') + + if not all([artifactory_url_var, artifactory_user_var, artifactory_apikey_var]): + exit_with_error("Missing credential environment variable names") + + artifactory_url = os.environ.get(artifactory_url_var) + artifactory_user = os.environ.get(artifactory_user_var) + artifactory_apikey = os.environ.get(artifactory_apikey_var) + + # Get required parameters + repository = os.environ.get('ARTIFACTORY_REPOSITORY') + target_path = os.environ.get('TARGET_PATH') + build_name = os.environ.get('BUILD_NAME') + build_number = os.environ.get('BUILD_NUMBER') + + # Fixed artifact path - always upload Shipyard environment data + artifact_path = '/tmp/shipyard_environment_data.json' + + # Validate required parameters + if not all([artifactory_url, repository, target_path]): + exit_with_error("Missing required parameters: ARTIFACTORY_URL, ARTIFACTORY_REPOSITORY, TARGET_PATH") + + if not all([artifactory_user, artifactory_apikey]): + exit_with_error("Artifactory credentials not found in environment variables") + + # Check if Shipyard data exists, create minimal data if not + if not os.path.exists(artifact_path): + print("⚠️ No Shipyard environment data found - creating minimal build data") + minimal_data = { + "project": os.environ.get('CIRCLE_PROJECT_REPONAME', 'unknown'), + "build": os.environ.get('CIRCLE_BUILD_NUM', '1'), + "commit": os.environ.get('CIRCLE_SHA1', ''), + "branch": os.environ.get('CIRCLE_BRANCH', ''), + "build_url": os.environ.get('CIRCLE_BUILD_URL', ''), + "created_at": datetime.utcnow().isoformat() + 'Z' + } + with open(artifact_path, 'w') as f: + json.dump(minimal_data, f, indent=2) + + # Set default values + build_name = build_name or os.environ.get('CIRCLE_PROJECT_REPONAME', 'unknown') + build_number = build_number or os.environ.get('CIRCLE_BUILD_NUM', '1') + + print(f"🚀 Starting JFrog upload process...") + print(f"Repository: {repository}") + print(f"Build: {build_name} #{build_number}") + + # Calculate checksums + checksums = calculate_checksums(artifact_path) + + # Read Shipyard environment data + shipyard_data = read_shipyard_data() + + # Create build properties + properties = create_build_properties(shipyard_data) + + # Upload artifact + if not upload_artifact(artifactory_url, artifactory_user, artifactory_apikey, + repository, artifact_path, target_path, checksums): + sys.exit(1) + + # Upload build information + upload_build_info(artifactory_url, artifactory_user, artifactory_apikey, + build_name, build_number, target_path, checksums, properties) + + print("🎉 Upload process completed!") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/scripts/orb.py b/src/scripts/orb.py index a3ad13e..f6bc2fc 100644 --- a/src/scripts/orb.py +++ b/src/scripts/orb.py @@ -18,6 +18,7 @@ import os import sys import time +import json from datetime import datetime from datetime import timedelta @@ -191,6 +192,21 @@ def main(): )) print(f'Shipyard environment {environment_id} data written to {bash_env_path}!') + + # Save the complete environment data as JSON for other scripts to use + json_data = { + "environment_id": environment_id, + "environment_data": environment_data, + "additional_urls": additional_urls, + "commit_hash": commit_hash, + "fetched_at": datetime.now().isoformat() + } + + json_file_path = '/tmp/shipyard_environment_data.json' + with open(json_file_path, 'w') as json_file: + json.dump(json_data, json_file, indent=2, default=str) + + print(f'Complete environment data saved to {json_file_path}') if __name__ == "__main__": diff --git a/src/scripts/orb.sh b/src/scripts/orb.sh deleted file mode 100644 index e8a882e..0000000 --- a/src/scripts/orb.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env sh - -# Check if sudo available -if [ "$(id -u)" = 0 ]; then export SUDO=""; else # Check if we are root - export SUDO="sudo"; -fi - -# Fix Cert error - https://www.omgubuntu.co.uk/2017/08/fix-google-gpg-key-linux-repository-error -wget -q -O /usr/share/keyrings/google-keyring.gpg https://dl.google.com/linux/linux_signing_key.pub -echo "deb [signed-by=/usr/share/keyrings/google-keyring.gpg] https://dl.google.com/linux/chrome/deb/ stable main" | $SUDO tee /etc/apt/sources.list.d/google-chrome.list > /dev/null - -# Install Python -if ! which python3 --version > /dev/null; then - echo "Trying to install Python..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3 python3-six apt-utils > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y python3 python3-six > /dev/null && \ - echo Installed! - - $SUDO ln -sf /usr/bin/python3 /usr/bin/python > /dev/null -fi - -# Install pip -if ! which pip > /dev/null; then - echo "Trying to install pip..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3-pip > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y python3-pip > /dev/null && \ - echo Installed! - - $SUDO ln -sf /usr/bin/pip3 /usr/bin/pip > /dev/null -fi - -# Check if python3-venv is installed, if not, install it -if ! dpkg -l | grep -q python3-venv; then - echo "Installing python3-venv..." - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3-venv > /dev/null && \ - echo "python3-venv installed!" - - which yum > /dev/null && \ - yum install -y python3-venv > /dev/null && \ - echo "python3-venv installed!" -fi - -# Install wget -if ! which wget > /dev/null; then - echo "Trying to install wget..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq wget > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y wget > /dev/null && \ - echo Installed! -fi - -# Download the orb -cd /tmp || exit - -wget -q https://github.com/shipyardbuild/circleci-orb/archive/refs/heads/master.tar.gz -tar xvzf master.tar.gz > /dev/null - -cd /tmp/circleci-orb-master/src/scripts || exit - -# Create a virtual environment -python3 -m venv /tmp/orb_env - -# Activate the virtual environment -# shellcheck disable=SC1091 -. /tmp/orb_env/bin/activate - -# Install the required packages -pip install -r requirements.txt > /dev/null - -# Run the orb -python orb.py diff --git a/src/scripts/requirements.txt b/src/scripts/requirements.txt index bafdc07..1a6dd3e 100644 --- a/src/scripts/requirements.txt +++ b/src/scripts/requirements.txt @@ -3,3 +3,6 @@ six >= 1.10 python_dateutil >= 2.5.3 setuptools >= 21.0.0 urllib3 >= 1.15.1 +requests >= 2.20.0 +pytest >= 6.0.0 +responses >= 0.18.0 diff --git a/src/scripts/setup-python.sh b/src/scripts/setup-python.sh new file mode 100644 index 0000000..c504d58 --- /dev/null +++ b/src/scripts/setup-python.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env sh + +# Check if sudo available +if [ "$(id -u)" = 0 ]; then export SUDO=""; else # Check if we are root + export SUDO="sudo"; +fi + +# Fix Cert error - https://www.omgubuntu.co.uk/2017/08/fix-google-gpg-key-linux-repository-error +if ! $SUDO wget -q -O /usr/share/keyrings/google-keyring.gpg https://dl.google.com/linux/linux_signing_key.pub; then + echo "Failed to download Google signing key" + exit 1 +fi +echo "deb [signed-by=/usr/share/keyrings/google-keyring.gpg] https://dl.google.com/linux/chrome/deb/ stable main" | $SUDO tee /etc/apt/sources.list.d/google-chrome.list > /dev/null + +# Run apt-get update once if needed +if which apt-get > /dev/null; then + $SUDO apt-get update -qq > /dev/null +fi + +# Install Python +if ! which python3 --version > /dev/null; then + echo "Trying to install Python..." + + if which apt-get > /dev/null; then + $SUDO apt-get install -qq python3 python3-six apt-utils > /dev/null && echo "Python installed!" + elif which yum > /dev/null; then + $SUDO yum install -y python3 python3-six > /dev/null && echo "Python installed!" + fi + + $SUDO ln -sf /usr/bin/python3 /usr/bin/python > /dev/null +fi + +# Install pip +if ! which pip > /dev/null; then + echo "Trying to install pip..." + + if which apt-get > /dev/null; then + $SUDO apt-get install -qq python3-pip > /dev/null && echo "pip installed!" + elif which yum > /dev/null; then + $SUDO yum install -y python3-pip > /dev/null && echo "pip installed!" + fi + + $SUDO ln -sf /usr/bin/pip3 /usr/bin/pip > /dev/null +fi + +# Check if python3-venv is installed, if not, install it +if ! python3 -m venv --help > /dev/null 2>&1; then + echo "Installing python3-venv..." + if which apt-get > /dev/null; then + $SUDO apt-get install -qq python3-venv > /dev/null && echo "python3-venv installed!" + elif which yum > /dev/null; then + $SUDO yum install -y python3-venv > /dev/null && echo "python3-venv installed!" + fi +fi + +# Install wget +if ! which wget > /dev/null; then + echo "Trying to install wget..." + + if which apt-get > /dev/null; then + $SUDO apt-get install -qq wget > /dev/null && echo "wget installed!" + elif which yum > /dev/null; then + $SUDO yum install -y wget > /dev/null && echo "wget installed!" + fi +fi + +echo "Python environment setup complete!" \ No newline at end of file diff --git a/src/scripts/setup-shell.sh b/src/scripts/setup-shell.sh new file mode 100755 index 0000000..c4412f1 --- /dev/null +++ b/src/scripts/setup-shell.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env sh +ls /tmp -la +BRANCH_NAME=feat/jfrog-integration + +# Convert branch name to archive filename format (replace / with -) +ARCHIVE_NAME=$(echo "${BRANCH_NAME}" | sed 's/\//-/g') + +# Download the orb (for dependencies only) +cd /tmp || exit + +if ! wget -q "https://github.com/shipyardbuild/circleci-orb/archive/refs/heads/${BRANCH_NAME}.tar.gz" -O "${ARCHIVE_NAME}.tar.gz"; then + echo "Failed to download orb from GitHub" + exit 1 +fi +ls -la +if ! tar xvzf "${ARCHIVE_NAME}.tar.gz" > /dev/null; then + echo "Failed to extract orb archive" + exit 1 +fi +ls -la +cd "/tmp/circleci-orb-${ARCHIVE_NAME}/src/scripts" || exit + +# Create a virtual environment +if ! python3 -m venv /tmp/orb_env; then + echo "Failed to create virtual environment" + exit 1 +fi + +# Activate the virtual environment +# shellcheck disable=SC1091 +. /tmp/orb_env/bin/activate + +# Install the required packages +pip install -r requirements.txt > /dev/null \ No newline at end of file From f1ce507ed388138425ca0f0c1f13fbfcdf143997 Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 11:02:16 -0400 Subject: [PATCH 2/6] chore: fix lint issue --- src/commands/upload-to-jfrog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/upload-to-jfrog.yml b/src/commands/upload-to-jfrog.yml index 879d33b..44d3a46 100644 --- a/src/commands/upload-to-jfrog.yml +++ b/src/commands/upload-to-jfrog.yml @@ -58,4 +58,4 @@ steps: TARGET_PATH: <> ENVIRONMENT_URL: <> command: | - python jfrog_upload.py \ No newline at end of file + python jfrog_upload.py From db02ea35a83f04f3a00b8cde6e1e76837ccca24d Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 11:14:02 -0400 Subject: [PATCH 3/6] chore: make install script more resilient --- src/scripts/setup-python.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/scripts/setup-python.sh b/src/scripts/setup-python.sh index c504d58..caa0487 100644 --- a/src/scripts/setup-python.sh +++ b/src/scripts/setup-python.sh @@ -1,20 +1,18 @@ #!/usr/bin/env sh +# Set environment variables to prevent interactive prompts +export DEBIAN_FRONTEND=noninteractive +export NEEDRESTART_MODE=a + # Check if sudo available if [ "$(id -u)" = 0 ]; then export SUDO=""; else # Check if we are root export SUDO="sudo"; fi -# Fix Cert error - https://www.omgubuntu.co.uk/2017/08/fix-google-gpg-key-linux-repository-error -if ! $SUDO wget -q -O /usr/share/keyrings/google-keyring.gpg https://dl.google.com/linux/linux_signing_key.pub; then - echo "Failed to download Google signing key" - exit 1 -fi -echo "deb [signed-by=/usr/share/keyrings/google-keyring.gpg] https://dl.google.com/linux/chrome/deb/ stable main" | $SUDO tee /etc/apt/sources.list.d/google-chrome.list > /dev/null - -# Run apt-get update once if needed +# Run apt-get update once if needed (skip Google Chrome setup as it's not needed for Python) if which apt-get > /dev/null; then - $SUDO apt-get update -qq > /dev/null + echo "Updating package lists..." + $SUDO apt-get update -qq > /dev/null 2>&1 || echo "Warning: apt-get update had some issues, continuing..." fi # Install Python @@ -22,9 +20,9 @@ if ! which python3 --version > /dev/null; then echo "Trying to install Python..." if which apt-get > /dev/null; then - $SUDO apt-get install -qq python3 python3-six apt-utils > /dev/null && echo "Python installed!" + $SUDO apt-get install -y -qq --no-install-recommends python3 python3-six apt-utils > /dev/null 2>&1 && echo "Python installed!" elif which yum > /dev/null; then - $SUDO yum install -y python3 python3-six > /dev/null && echo "Python installed!" + $SUDO yum install -y python3 python3-six > /dev/null 2>&1 && echo "Python installed!" fi $SUDO ln -sf /usr/bin/python3 /usr/bin/python > /dev/null @@ -35,9 +33,9 @@ if ! which pip > /dev/null; then echo "Trying to install pip..." if which apt-get > /dev/null; then - $SUDO apt-get install -qq python3-pip > /dev/null && echo "pip installed!" + $SUDO apt-get install -y -qq --no-install-recommends python3-pip > /dev/null 2>&1 && echo "pip installed!" elif which yum > /dev/null; then - $SUDO yum install -y python3-pip > /dev/null && echo "pip installed!" + $SUDO yum install -y python3-pip > /dev/null 2>&1 && echo "pip installed!" fi $SUDO ln -sf /usr/bin/pip3 /usr/bin/pip > /dev/null @@ -47,9 +45,9 @@ fi if ! python3 -m venv --help > /dev/null 2>&1; then echo "Installing python3-venv..." if which apt-get > /dev/null; then - $SUDO apt-get install -qq python3-venv > /dev/null && echo "python3-venv installed!" + $SUDO apt-get install -y -qq --no-install-recommends python3-venv > /dev/null 2>&1 && echo "python3-venv installed!" elif which yum > /dev/null; then - $SUDO yum install -y python3-venv > /dev/null && echo "python3-venv installed!" + $SUDO yum install -y python3-venv > /dev/null 2>&1 && echo "python3-venv installed!" fi fi @@ -58,9 +56,9 @@ if ! which wget > /dev/null; then echo "Trying to install wget..." if which apt-get > /dev/null; then - $SUDO apt-get install -qq wget > /dev/null && echo "wget installed!" + $SUDO apt-get install -y -qq --no-install-recommends wget > /dev/null 2>&1 && echo "wget installed!" elif which yum > /dev/null; then - $SUDO yum install -y wget > /dev/null && echo "wget installed!" + $SUDO yum install -y wget > /dev/null 2>&1 && echo "wget installed!" fi fi From c6cb2eb783fff78e8ad7ecf90b0697f3beff1e45 Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 11:31:34 -0400 Subject: [PATCH 4/6] fix: use virtualenv --- src/commands/fetch-shipyard-env.yml | 6 ++++++ src/commands/upload-to-jfrog.yml | 14 +++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/commands/fetch-shipyard-env.yml b/src/commands/fetch-shipyard-env.yml index 248aba2..856ab66 100644 --- a/src/commands/fetch-shipyard-env.yml +++ b/src/commands/fetch-shipyard-env.yml @@ -45,4 +45,10 @@ steps: SHIPYARD_TIMEOUT: <> SHIPYARD_APP_NAME: <> command: | + # Activate the virtual environment + . /tmp/orb_env/bin/activate + + # Change to the directory where orb.py is located + cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit + python orb.py diff --git a/src/commands/upload-to-jfrog.yml b/src/commands/upload-to-jfrog.yml index 44d3a46..b0dd2e9 100644 --- a/src/commands/upload-to-jfrog.yml +++ b/src/commands/upload-to-jfrog.yml @@ -37,15 +37,7 @@ parameters: default: "shipyard/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}/environment-data.json" description: "Target path in JFrog repository for Shipyard environment data" - environment-url: - type: string - default: "${SHIPYARD_ENVIRONMENT_URL}" - description: "Shipyard environment URL to include in build evidence" - steps: - - run: - name: "Setup shell" - command: <> - run: name: "Upload artifact to JFrog with build evidence" environment: @@ -56,6 +48,10 @@ steps: BUILD_NAME: <> BUILD_NUMBER: <> TARGET_PATH: <> - ENVIRONMENT_URL: <> command: | + # Activate the virtual environment + . /tmp/orb_env/bin/activate + + # Change to the directory where orb.py is located + cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit python jfrog_upload.py From 24204f2a156ec904fca82a262cdde1df4fc1b0b9 Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 11:47:52 -0400 Subject: [PATCH 5/6] fix: lint issues --- src/commands/fetch-shipyard-env.yml | 2 -- src/commands/upload-to-jfrog.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/src/commands/fetch-shipyard-env.yml b/src/commands/fetch-shipyard-env.yml index 856ab66..a900c21 100644 --- a/src/commands/fetch-shipyard-env.yml +++ b/src/commands/fetch-shipyard-env.yml @@ -47,8 +47,6 @@ steps: command: | # Activate the virtual environment . /tmp/orb_env/bin/activate - # Change to the directory where orb.py is located cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit - python orb.py diff --git a/src/commands/upload-to-jfrog.yml b/src/commands/upload-to-jfrog.yml index b0dd2e9..0d8575a 100644 --- a/src/commands/upload-to-jfrog.yml +++ b/src/commands/upload-to-jfrog.yml @@ -51,7 +51,6 @@ steps: command: | # Activate the virtual environment . /tmp/orb_env/bin/activate - # Change to the directory where orb.py is located cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit python jfrog_upload.py From 74d86c8b9a33c1d11027db08a868dd21d21d97c0 Mon Sep 17 00:00:00 2001 From: Akshay Kalia Date: Mon, 4 Aug 2025 11:52:46 -0400 Subject: [PATCH 6/6] move commands to script files --- src/commands/fetch-shipyard-env.yml | 7 +-- src/commands/upload-to-jfrog.yml | 8 +-- src/scripts/orb.sh | 90 ----------------------------- src/scripts/run-jfrog-upload.sh | 10 ++++ src/scripts/run-orb.sh | 10 ++++ 5 files changed, 23 insertions(+), 102 deletions(-) delete mode 100644 src/scripts/orb.sh create mode 100644 src/scripts/run-jfrog-upload.sh create mode 100644 src/scripts/run-orb.sh diff --git a/src/commands/fetch-shipyard-env.yml b/src/commands/fetch-shipyard-env.yml index a900c21..6eeb711 100644 --- a/src/commands/fetch-shipyard-env.yml +++ b/src/commands/fetch-shipyard-env.yml @@ -44,9 +44,4 @@ steps: environment: SHIPYARD_TIMEOUT: <> SHIPYARD_APP_NAME: <> - command: | - # Activate the virtual environment - . /tmp/orb_env/bin/activate - # Change to the directory where orb.py is located - cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit - python orb.py + command: <> diff --git a/src/commands/upload-to-jfrog.yml b/src/commands/upload-to-jfrog.yml index 0d8575a..5370fd1 100644 --- a/src/commands/upload-to-jfrog.yml +++ b/src/commands/upload-to-jfrog.yml @@ -48,9 +48,5 @@ steps: BUILD_NAME: <> BUILD_NUMBER: <> TARGET_PATH: <> - command: | - # Activate the virtual environment - . /tmp/orb_env/bin/activate - # Change to the directory where orb.py is located - cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit - python jfrog_upload.py + ENVIRONMENT_URL: <> + command: <> diff --git a/src/scripts/orb.sh b/src/scripts/orb.sh deleted file mode 100644 index f7145ab..0000000 --- a/src/scripts/orb.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env sh - -# Check if sudo available -if [ "$(id -u)" = 0 ]; then export SUDO=""; else # Check if we are root - export SUDO="sudo"; -fi - -# Fix Cert error - https://www.omgubuntu.co.uk/2017/08/fix-google-gpg-key-linux-repository-error -$SUDO wget -q -O /usr/share/keyrings/google-keyring.gpg https://dl.google.com/linux/linux_signing_key.pub -echo "deb [signed-by=/usr/share/keyrings/google-keyring.gpg] https://dl.google.com/linux/chrome/deb/ stable main" | $SUDO tee /etc/apt/sources.list.d/google-chrome.list > /dev/null - -# Install Python -if ! which python3 --version > /dev/null; then - echo "Trying to install Python..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3 python3-six apt-utils > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y python3 python3-six > /dev/null && \ - echo Installed! - - $SUDO ln -sf /usr/bin/python3 /usr/bin/python > /dev/null -fi - -# Install pip -if ! which pip > /dev/null; then - echo "Trying to install pip..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3-pip > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y python3-pip > /dev/null && \ - echo Installed! - - $SUDO ln -sf /usr/bin/pip3 /usr/bin/pip > /dev/null -fi - -# Check if python3-venv is installed, if not, install it -if ! dpkg -l | grep -q python3-venv; then - echo "Installing python3-venv..." - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq python3-venv > /dev/null && \ - echo "python3-venv installed!" - - which yum > /dev/null && \ - yum install -y python3-venv > /dev/null && \ - echo "python3-venv installed!" -fi - -# Install wget -if ! which wget > /dev/null; then - echo "Trying to install wget..." - - which apt-get > /dev/null && \ - $SUDO apt-get update -qq > /dev/null && \ - $SUDO apt-get install -qq wget > /dev/null && \ - echo Installed! - - which yum > /dev/null && \ - yum install -y wget > /dev/null && \ - echo Installed! -fi - -# Download the orb -cd /tmp || exit - -wget -q https://github.com/shipyardbuild/circleci-orb/archive/refs/heads/master.tar.gz -tar xvzf master.tar.gz > /dev/null - -cd /tmp/circleci-orb-master/src/scripts || exit - -# Create a virtual environment -python3 -m venv /tmp/orb_env - -# Activate the virtual environment -# shellcheck disable=SC1091 -. /tmp/orb_env/bin/activate - -# Install the required packages -pip install -r requirements.txt > /dev/null - -# Run the orb -python orb.py diff --git a/src/scripts/run-jfrog-upload.sh b/src/scripts/run-jfrog-upload.sh new file mode 100644 index 0000000..a91f81f --- /dev/null +++ b/src/scripts/run-jfrog-upload.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +# Activate the virtual environment +. /tmp/orb_env/bin/activate + +# Change to the directory where jfrog_upload.py is located +cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit + +# Run the JFrog upload script +python jfrog_upload.py \ No newline at end of file diff --git a/src/scripts/run-orb.sh b/src/scripts/run-orb.sh new file mode 100644 index 0000000..f5a7719 --- /dev/null +++ b/src/scripts/run-orb.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +# Activate the virtual environment +. /tmp/orb_env/bin/activate + +# Change to the directory where orb.py is located +cd /tmp/circleci-orb-feat-jfrog-integration/src/scripts || exit + +# Run the orb +python orb.py \ No newline at end of file