CI: create merge and upload composite action

This commit extracts merge-upload job steps to a composite action to avoid repetition, make changes easier Signed-off-by: Birol Bilgin <[email protected]>
DataDog · Oct 16, 2024 · 98ada1a · 98ada1a
1 parent 63cd391
commit 98ada1a
Show file tree

Hide file tree

Showing 13 changed files with 165 additions and 396 deletions.
diff --git a/.github/actions/merge-artifacts/action.yaml b/.github/actions/merge-artifacts/action.yaml
@@ -0,0 +1,57 @@
+name: Merge and Upload Artifacts
+description: Cheks artifacts and if they exists merges and uploads them
+
+inputs:
+  name:
+    required: true
+    description: 'The name of the artifact that the artifacts will be merged into'
+  token:
+    required: true
+    description: 'GITHUB_TOKEN for accessing to github api'
+  pattern:
+    required: true
+    description: 'A glob pattern matching the artifacts that should be merged'
+  delete-merged:
+    required: true
+    description: 'If true, the artifacts that were merged will be deleted'
+    default: 'true'
+
+runs:
+  using: composite
+  steps:
+    - name: List All Artifacts
+      id: list_artifacts
+      shell: bash
+      env:
+        ARTIFACTS_FILE: "artifacts_list_${{ github.run_id }}_${{ github.run_number }}.json"
+      run: |
+        echo "artifacts_file=${{ env.ARTIFACTS_FILE }}" >> $GITHUB_OUTPUT
+        if [ ! -f ${{ env.ARTIFACTS_FILE }} ];then
+          echo "Fetching artifacts"
+          curl --fail --show-error --silent \
+                -H "Authorization: token ${{ inputs.token }}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
+                | jq -r '.artifacts[] | .name' > ${{ env.ARTIFACTS_FILE }}
+        fi
+    - name: Filter Artifacts
+      id: filter_artifacts
+      shell: bash
+      run: |
+        pattern="${{ inputs.pattern }}"
+        regex="^${pattern//\*/.*}$"
+        echo "Filtered Artifacts starts with ${{ inputs.pattern }}"
+        if grep -E "$regex" ${{ steps.list_artifacts.outputs.artifacts_file }}; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "::warning::No matching ${{ inputs.pattern }} artifacts found."
+        fi
+    - name: Merge ${{ inputs.name }}
+      if: ${{ steps.filter_artifacts.outputs.exists == 'true' }}
+      uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      with:
+        name: ${{ inputs.name }}
+        pattern: ${{ inputs.pattern }}
+        retention-days: 5
+        delete-merged: ${{ inputs.delete-merged }}
+
diff --git a/.github/workflows/conformance-aks.yaml b/.github/workflows/conformance-aks.yaml
@@ -355,47 +355,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: installation-and-connectivity
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}

diff --git a/.github/workflows/conformance-aws-cni.yaml b/.github/workflows/conformance-aws-cni.yaml
@@ -351,47 +351,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: installation-and-connectivity
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}

diff --git a/.github/workflows/conformance-clustermesh.yaml b/.github/workflows/conformance-clustermesh.yaml
@@ -636,47 +636,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: installation-and-connectivity
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}

diff --git a/.github/workflows/conformance-eks.yaml b/.github/workflows/conformance-eks.yaml
@@ -403,47 +403,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: installation-and-connectivity
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}

diff --git a/.github/workflows/conformance-externalworkloads.yaml b/.github/workflows/conformance-externalworkloads.yaml
@@ -415,47 +415,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: installation-and-connectivity
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}

diff --git a/.github/workflows/conformance-ginkgo.yaml b/.github/workflows/conformance-ginkgo.yaml
@@ -478,47 +478,23 @@ jobs:
     runs-on: ubuntu-latest
     needs: setup-and-test
     steps:
-      # TODO remove list and filter artifact steps if this PR https://github.com/actions/upload-artifact/pull/521 is merged
-      - name: List All Artifacts
-        id: list_artifacts
-        run: |
-          echo "Fetching artifacts"
-          curl --fail --show-error --silent \
-               -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
-               | jq -r '.artifacts[] | .name' > artifacts_list.json
-      - name: Filter Artifacts
-        id: filter_artifacts
-        run: |
-          echo "Filtered Artifacts starts with cilium-sysdumps:"
-          if grep -E "^cilium-sysdumps-" artifacts_list.json; then
-            echo "sysdumps=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-sysdumps-* artifacts found."
-          fi
-          echo "Filtered Artifacts starts with cilium-junits:"
-          if grep -E "^cilium-junits-" artifacts_list.json; then
-            echo "junits=true" >> $GITHUB_OUTPUT
-          else
-            echo "::warning::No matching cilium-junits-* artifacts found."
-          fi
+      - name: Checkout context ref (trusted)
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.context-ref || github.sha }}
+          persist-credentials: false
       - name: Merge Sysdumps
-        if: ${{ steps.filter_artifacts.outputs.sysdumps == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-sysdumps
           pattern: cilium-sysdumps-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Merge JUnits
-        if: ${{ steps.filter_artifacts.outputs.junits == 'true' }}
-        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: ./.github/actions/merge-artifacts
         with:
           name: cilium-junits
           pattern: cilium-junits-*
-          retention-days: 5
-          delete-merged: true
+          token: ${{ secrets.GITHUB_TOKEN }}
 
   commit-status-final:
     if: ${{ always() }}