From ae28de7a7493cbc5afd50695c85fe1f3c41c376d Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 11:29:27 +0530 Subject: [PATCH 1/9] feat: add pregen to setup environment Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index e0f35728a2..b8037c923f 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -19,8 +19,22 @@ concurrency: jobs: + pregen: + runs-on: ubuntu-latest + outputs: + venv_path: ${{ steps.set_env.outputs.venv_path }} + steps: + - name: Create venv & install prerequisites + id: set_env + run: | + python3.12 -m venv .venv + source .venv/bin/activate + pip install wheel setuptools + echo "::set-output name=venv_path::$(pwd)/.venv" + test_non_dotnet: runs-on: ["snapshot-test"] + needs: pregen steps: - uses: actions/checkout@v4 @@ -33,13 +47,12 @@ jobs: env: SHELL: bash run: | + source ${{ needs.pregen.outputs.venv_path }}/bin/activate rm -rf original_snapshots new_snapshots src_repos cdxgen_tarball=$(pnpm pack | tail -1) sudo npm install -g "$cdxgen_tarball" git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots - python3.12 -m venv .venv - source .venv/bin/activate && pip install setuptools wheel - source .venv/bin/activate && pip install -r test/diff/requirements.txt + pip install -r test/diff/requirements.txt - name: Generate scripts run: | From 570b5dd7dbd65003b3aad993af51df7369ec46cf Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 11:50:09 +0530 Subject: [PATCH 2/9] fix: updated the use of env Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index b8037c923f..29e71f0ec6 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -24,12 +24,12 @@ jobs: outputs: venv_path: ${{ steps.set_env.outputs.venv_path }} steps: - - name: Create venv & install prerequisites + - name: Create venv and install prerequisites id: set_env run: | python3.12 -m venv .venv source .venv/bin/activate - pip install wheel setuptools + pip install setuptools wheel echo "::set-output name=venv_path::$(pwd)/.venv" test_non_dotnet: @@ -42,21 +42,21 @@ jobs: - name: Set up pnpm uses: pnpm/action-setup@v4 - - name: cdxgen, custom-json-diff installs - shell: bash - env: - SHELL: bash - run: | - source ${{ needs.pregen.outputs.venv_path }}/bin/activate - rm -rf original_snapshots new_snapshots src_repos - cdxgen_tarball=$(pnpm pack | tail -1) - sudo npm install -g "$cdxgen_tarball" - git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots - pip install -r test/diff/requirements.txt + - name: cdxgen, custom-json-diff installs + shell: bash + env: + SHELL: bash + run: | + rm -rf original_snapshots new_snapshots src_repos + cdxgen_tarball=$(pnpm pack | tail -1) + sudo npm install -g "$cdxgen_tarball" + git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots + source ${{ needs.pregen.outputs.venv_path }}/bin/activate + pip install -r test/diff/requirements.txt - name: Generate scripts run: | - source .venv/bin/activate + source ${{ needs.pregen.outputs.venv_path }}/bin/activate python test/diff/generate.py env: ATOM_JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 @@ -82,10 +82,10 @@ jobs: - name: Test BOMs run: | - source .venv/bin/activate + source ${{ needs.pregen.outputs.venv_path }}/bin/activate python test/diff/diff_tests.py --migrate-legacy if test -f new_snapshots/diffs.json; then - echo "status=FAILED" >> "$GITHUB_ENV" + echo "status=FAILED" >> $GITHUB_ENV fi - name: Upload diff From 66b1421287cd0dece454365fa48b1aa133d16978 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 11:50:09 +0530 Subject: [PATCH 3/9] fix: updated the use of env Signed-off-by: AnsahMohammad fix: updated the use of env Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 29e71f0ec6..236076b025 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -28,6 +28,7 @@ jobs: id: set_env run: | python3.12 -m venv .venv + echo "venv_path=$(pwd)/.venv" >> $GITHUB_ENV source .venv/bin/activate pip install setuptools wheel echo "::set-output name=venv_path::$(pwd)/.venv" @@ -60,6 +61,7 @@ jobs: python test/diff/generate.py env: ATOM_JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 + - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 with: From 61b089c2c988ad857332c831e4cf8733345be1b2 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 12:39:34 +0530 Subject: [PATCH 4/9] fix: added debug paths Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 236076b025..a9d1af4ff8 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -17,6 +17,10 @@ concurrency: group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}" cancel-in-progress: true +defaults: + run: + working-directory: ${{ github.workspace }} + jobs: pregen: @@ -24,25 +28,33 @@ jobs: outputs: venv_path: ${{ steps.set_env.outputs.venv_path }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Create venv and install prerequisites id: set_env run: | python3.12 -m venv .venv - echo "venv_path=$(pwd)/.venv" >> $GITHUB_ENV + echo "VENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV + echo "::set-output name=venv_path::$(pwd)/.venv" source .venv/bin/activate pip install setuptools wheel - echo "::set-output name=venv_path::$(pwd)/.venv" test_non_dotnet: runs-on: ["snapshot-test"] needs: pregen steps: - - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - name: Set up pnpm uses: pnpm/action-setup@v4 + - name: Debug paths + run: | + echo "VENV_PATH: ${{ needs.pregen.outputs.venv_path }}" + ls -al ${{ needs.pregen.outputs.venv_path }} + - name: cdxgen, custom-json-diff installs shell: bash env: From 60bab96f4d95fda074a2b0b4b66f91a799a2b570 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 12:51:41 +0530 Subject: [PATCH 5/9] fix: added debug paths Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index a9d1af4ff8..09ff981530 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -19,7 +19,7 @@ concurrency: defaults: run: - working-directory: ${{ github.workspace }} + shell: bash jobs: From c28a67809f6da2846152e86ee67927ecb31802ac Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 13:33:31 +0530 Subject: [PATCH 6/9] fix: added more debugs Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 09ff981530..de527f0c56 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -34,9 +34,11 @@ jobs: - name: Create venv and install prerequisites id: set_env run: | + echo "pwd:" + pwd + ls -al .venv python3.12 -m venv .venv echo "VENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV - echo "::set-output name=venv_path::$(pwd)/.venv" source .venv/bin/activate pip install setuptools wheel @@ -53,7 +55,10 @@ jobs: - name: Debug paths run: | echo "VENV_PATH: ${{ needs.pregen.outputs.venv_path }}" - ls -al ${{ needs.pregen.outputs.venv_path }} + echo "Running pwd to confirm the directory:" + pwd + ls -al ${{ needs.pregen.outputs.venv_path }} # Path access verification + - name: cdxgen, custom-json-diff installs shell: bash From 5dc52de2d04b6ca9836391e3f532afb4c6a11af8 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 13:36:28 +0530 Subject: [PATCH 7/9] fix: added more debugs 2 Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index de527f0c56..8abd2ce5ed 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -36,8 +36,8 @@ jobs: run: | echo "pwd:" pwd - ls -al .venv python3.12 -m venv .venv + ls -al .venv || echo venv doesn;t exist echo "VENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV source .venv/bin/activate pip install setuptools wheel @@ -57,7 +57,7 @@ jobs: echo "VENV_PATH: ${{ needs.pregen.outputs.venv_path }}" echo "Running pwd to confirm the directory:" pwd - ls -al ${{ needs.pregen.outputs.venv_path }} # Path access verification + ls -al ${{ needs.pregen.outputs.venv_path }} || oops no venv here - name: cdxgen, custom-json-diff installs From e62f7d897846f61fd405332ebc7371d09640f0fd Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 13:37:56 +0530 Subject: [PATCH 8/9] fix: added more debugs 3 nit Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 8abd2ce5ed..ea17b63036 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -37,7 +37,7 @@ jobs: echo "pwd:" pwd python3.12 -m venv .venv - ls -al .venv || echo venv doesn;t exist + ls -al .venv || echo venv doesnt exist echo "VENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV source .venv/bin/activate pip install setuptools wheel From 956781af0c11066412bb598905fdad837de95d54 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Sat, 25 Jan 2025 13:44:35 +0530 Subject: [PATCH 9/9] fix: added persistent .venv Signed-off-by: AnsahMohammad --- .github/workflows/snapshot-tests.yml | 35 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index ea17b63036..0e1f9ee20b 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -34,14 +34,19 @@ jobs: - name: Create venv and install prerequisites id: set_env run: | - echo "pwd:" - pwd python3.12 -m venv .venv - ls -al .venv || echo venv doesnt exist - echo "VENV_PATH=$(pwd)/.venv" >> $GITHUB_ENV + echo "venv_path=$(pwd)/.venv" >> $GITHUB_OUTPUT source .venv/bin/activate pip install setuptools wheel + - name: Cache venv + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + venv-${{ runner.os }}- + test_non_dotnet: runs-on: ["snapshot-test"] needs: pregen @@ -49,17 +54,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Restore venv + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + venv-${{ runner.os }}- + - name: Set up pnpm uses: pnpm/action-setup@v4 - - name: Debug paths - run: | - echo "VENV_PATH: ${{ needs.pregen.outputs.venv_path }}" - echo "Running pwd to confirm the directory:" - pwd - ls -al ${{ needs.pregen.outputs.venv_path }} || oops no venv here - - - name: cdxgen, custom-json-diff installs shell: bash env: @@ -69,12 +74,12 @@ jobs: cdxgen_tarball=$(pnpm pack | tail -1) sudo npm install -g "$cdxgen_tarball" git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots - source ${{ needs.pregen.outputs.venv_path }}/bin/activate + source .venv/bin/activate pip install -r test/diff/requirements.txt - name: Generate scripts run: | - source ${{ needs.pregen.outputs.venv_path }}/bin/activate + source .venv/bin/activate python test/diff/generate.py env: ATOM_JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 @@ -101,7 +106,7 @@ jobs: - name: Test BOMs run: | - source ${{ needs.pregen.outputs.venv_path }}/bin/activate + source .venv/bin/activate python test/diff/diff_tests.py --migrate-legacy if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> $GITHUB_ENV