🚀 arc runner migration patch 2.1 (#1242)

* 🚀 update `check_tests` pipeline to use org `runtime` runner Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 update `Run tests` pipeline with required changes for org arc runner migration Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 handle `polkadot` Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 use `bash` for `launch testnet` step Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 try docker Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 try docker Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 try docker Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 dasel don't need cargo Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 missing jq Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 address dasel Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 address dasel Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 address dasel Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 address dasel Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 address dasel Signed-off-by: Anathan (Ana) Ye <[email protected]> * 🐛 fix `calamari integration test` Signed-off-by: Anathan (Ana) Ye <[email protected]> * Fix calamari integration tests Signed-off-by: Dengjianping <[email protected]> * Fix storage insertion Signed-off-by: Dengjianping <[email protected]> * Remove unused cmd Signed-off-by: Dengjianping <[email protected]> * Do not skip compilation Signed-off-by: Dengjianping <[email protected]> --------- Signed-off-by: Anathan (Ana) Ye <[email protected]> Signed-off-by: Dengjianping <[email protected]> Co-authored-by: Dengjianping <[email protected]>
Manta-Network · Aug 22, 2023 · af50afb · af50afb
1 parent 0814f5d
commit af50afb
Show file tree

Hide file tree

Showing 16 changed files with 486 additions and 965 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+.github/ @Manta-Network/devops
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,6 +8,10 @@ Before we can approve this PR for merge, please make sure that **all** the follo
 - [ ] Added **one or more** labels from the `A-` and `C-` groups to this PR
 - [ ] Explicitly labelled `A-calamari` and/or `A-manta` if your changes are meant for/impact either of these (CI depends on it)
 - [ ] Re-reviewed `Files changed` in the Github PR explorer.
+- [ ] Add `A-integration-test-checks` to run **start-integration-test-checks** (Required)
+- [ ] Add `A-benchmark-checks` to run **start-benchmark-check** (Required)
+- [ ] Add `A-unit-test-checks` to run **start-unit-test-checks** (Required)
+- [ ] Add `A-congestion-test-checks` to run **start-integration-test-checks** (Optional)
 
 
 Situational Notes:

diff --git a/.github/workflows/check_tests.yml b/.github/workflows/check_tests.yml
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -46,7 +46,6 @@ jobs:
           image_id=`cat docker_id.log`
           echo $image_id
           echo "Adjusting permissions so we can access docker logs..."
-          sudo chmod -R +x /var/lib/docker/
           sudo cat /var/lib/docker/containers/${image_id}/${image_id}-json.log
       - uses: actions/checkout@v2
         with:

diff --git a/.github/workflows/integration_test_calamari.yml b/.github/workflows/integration_test_calamari.yml
diff --git a/.github/workflows/integration_test_manta.yml b/.github/workflows/integration_test_manta.yml
diff --git a/.github/workflows/label_checker.yml b/.github/workflows/label_checker.yml
@@ -12,3 +12,11 @@ jobs:
         with:
           one_of: L-added,L-changed,L-fixed,L-deprecated,L-removed,L-skip
           repo_token: ${{ secrets.GITHUB_TOKEN }}
+  check_a_labels:
+    name: A- Labels Check
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: docker://agilepathway/pull-request-label-checker:latest
+        with:
+          all_of: A-unit-test-checks,A-integration-test-checks,A-benchmark-checks
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml
@@ -10,16 +10,17 @@ on:
         description: The name of the chain under test. Usually, you would pass a dev chain
         default: manta-dev
         required: true
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 env:
   CHAIN: ${{github.event.inputs.chain}}
   ENDPOINT: ${{github.event.inputs.endpoint}}
   DEBIAN_FRONTEND: noninteractive
 jobs:
-  start-checks:
+  update-checks:
     timeout-minutes: 120
-    runs-on: runtime
-    container:
-      image: ubuntu:20.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         node-version: [16.x]
@@ -31,24 +32,15 @@ jobs:
           SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
           SCCACHE_VERSION: v0.5.3
         run: |
-          apt update
-          apt install -y curl
+          sudo apt update
+          sudo apt install -y curl
           SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
           mkdir -p $HOME/.local/bin
           curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
           mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
           chmod +x $HOME/.local/bin/sccache
           echo "$HOME/.local/bin" >> $GITHUB_PATH
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-      - name: cache cargo registry
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-          key: cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            cargo-registry-
       - name: cache sccache
         uses: actions/cache@v2
         continue-on-error: false
@@ -62,32 +54,16 @@ jobs:
       - name: init
         shell: bash
         run: |
-          apt update
-          apt install -y pkg-config libssl-dev protobuf-compiler curl build-essential clang git
+          sudo apt update
+          sudo apt install -y pkg-config libssl-dev protobuf-compiler curl build-essential clang git
           protoc --version
-          apt remove -y '^dotnet-.*'
-          apt remove -y '^mongodb-.*'
-          apt remove -y '^mysql-.*'
-          apt remove -y '^postgresql-*'
-          apt remove -y google-cloud-sdk google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri
-          apt autoremove -y
-          apt clean
-          rm -rf /usr/local/lib/android
-          rm -rf /usr/share/dotnet
           curl -s https://sh.rustup.rs -sSf | sh -s -- -y
           source ${HOME}/.cargo/env
           rustup toolchain install nightly-2023-03-03
           rustup default nightly-2023-03-03
           rustup target add wasm32-unknown-unknown
-      - name: cache cargo build
-        uses: actions/cache@v2
-        with:
-          path: |
-            ./target
-          key: cargo-build-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            cargo-build-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}
-            cargo-build-
+      - name: cache cargo
+        uses: Swatinem/rust-cache@v2
       - name: Build New Binary
         env:
           RUST_BACKTRACE: full
@@ -97,7 +73,7 @@ jobs:
         shell: bash
         run: |
           source ${HOME}/.cargo/env
-          RUSTC_BOOTSTRAP=1 cargo build --release
+          RUSTC_BOOTSTRAP=1 cargo build --release --timings
           cp target/release/manta $HOME/manta
           chmod +x $HOME/manta
       - name: Fetch and chmod old release binary
@@ -120,32 +96,32 @@ jobs:
           path: zombienet-tool
       - name: create launch config
         run: |
-          ls -ahl ${{ github.workspace }}/zombienet/tests
-          cd ${{ github.workspace }}
+          ls -ahl $GITHUB_WORKSPACE/zombienet/tests
+          cd $GITHUB_WORKSPACE
           dasel put -t string -v $HOME/polkadot -r toml \
-            -f ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml \
+            -f $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml \
             '.relaychain.default_command'
           dasel put -t string -v $HOME/manta -r toml \
-            -f ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml \
+            -f $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml \
             '.parachains.[0].collators.[0].command'
           if [[ "$CHAIN" == *"manta"* ]]; then
             dasel put -t int -v 2104 -r toml \
-              -f ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml \
+              -f $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml \
               '.parachains.[0].id'
             dasel put -t string -v $CHAIN -r toml \
-              -f ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml \
+              -f $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml \
               '.parachains.[0].chain'
           fi;
-          cat ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml
+          cat $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml
       - name: launch testnet
         run: |
-          cd ${{ github.workspace }}/zombienet-tool/javascript
+          cd $GITHUB_WORKSPACE/zombienet-tool/javascript
           yarn
           npm run build
           pm2 start packages/cli/dist/cli.js --name zombienet \
-            --output ${{ github.workspace }}/stdout.log \
-            --error ${{ github.workspace }}/stderr.log \
-            -- spawn --provider native ${{ github.workspace }}/zombienet/tests/0003-metadata-diff.toml
+            --output $GITHUB_WORKSPACE/stdout.log \
+            --error $GITHUB_WORKSPACE/stderr.log \
+            -- spawn --provider native $GITHUB_WORKSPACE/zombienet/tests/0003-metadata-diff.toml
       - name: Prepare output
         run: |
           VERSION=$($HOME/manta --version)
@@ -161,15 +137,15 @@ jobs:
       - name: Compare the metadata
         run: |
           echo $ENDPOINT
-          cd ${{ github.workspace }}/tools
+          cd $GITHUB_WORKSPACE/tools
           yarn
           CMD="yarn run:metadata $ENDPOINT ws://127.0.0.1:9921"
           echo -e "Running:\n$CMD"
           $CMD >> output.txt
           sed -z -i 's/\n\n/\n/g' output.txt
           cat output.txt | egrep -n -i ''
-          chmod +x ${{ github.workspace }}/scripts/github/extrinsic-ordering-filter.sh
-          SUMMARY=$(${{ github.workspace }}/scripts/github/extrinsic-ordering-filter.sh output.txt)
+          chmod +x $GITHUB_WORKSPACE/scripts/github/extrinsic-ordering-filter.sh
+          SUMMARY=$($GITHUB_WORKSPACE/scripts/github/extrinsic-ordering-filter.sh output.txt)
           echo -e $SUMMARY
           echo -e $SUMMARY >> output.txt
       - name: Show result

diff --git a/.github/workflows/publish_draft_releases.yml b/.github/workflows/publish_draft_releases.yml
@@ -5,14 +5,14 @@ on:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 env:
   DEBIAN_FRONTEND: noninteractive
 jobs:
   get-rust-versions:
     if: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     container:
       image: paritytech/ci-linux:production
     outputs:
@@ -28,11 +28,9 @@ jobs:
   build-runtimes:
     timeout-minutes: 180
     if: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     env:
       CARGO_TERM_COLOR: always
-    container:
-      image: ubuntu:20.04
     strategy:
       matrix:
         runtime:
@@ -43,7 +41,7 @@ jobs:
       - name: cache target dir
         uses: actions/cache@v2
         with:
-          path: "${{ github.workspace }}/runtime/${{ matrix.runtime.name }}/target"
+          path: "$GITHUB_WORKSPACE/runtime/${{ matrix.runtime.name }}/target"
           key: srtool-target-${{ matrix.runtime.name }}-${{ github.sha }}
           restore-keys: |
             srtool-target-${{ matrix.runtime.name }}
@@ -75,9 +73,7 @@ jobs:
   build-node-current:
     timeout-minutes: 180
     if: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    runs-on: runtime
-    container:
-      image: ubuntu:20.04
+    runs-on: ubuntu-20.04
     env:
       CARGO_TERM_COLOR: always
     steps:
@@ -87,24 +83,15 @@ jobs:
           SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
           SCCACHE_VERSION: v0.5.3
         run: |
-          apt update
-          apt install -y curl
+          sudo apt update
+          sudo apt install -y curl
           SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
           mkdir -p $HOME/.local/bin
           curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
           mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
           chmod +x $HOME/.local/bin/sccache
           echo "$HOME/.local/bin" >> $GITHUB_PATH
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-      - name: cache cargo registry
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-          key: cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            cargo-
       - name: cache sccache
         uses: actions/cache@v2
         continue-on-error: false
@@ -118,8 +105,8 @@ jobs:
       - name: init
         shell: bash
         run: |
-          apt update
-          apt install -y pkg-config libssl-dev protobuf-compiler curl build-essential clang git
+          sudo apt update
+          sudo apt install -y pkg-config libssl-dev protobuf-compiler curl build-essential clang git
           protoc --version
           curl -s https://sh.rustup.rs -sSf | sh -s -- -y
           source ${HOME}/.cargo/env
@@ -129,6 +116,8 @@ jobs:
           rustup target add wasm32-unknown-unknown
           rustup install 1.69.0
           rustup target add wasm32-unknown-unknown --toolchain 1.69.0
+      - name: cache cargo
+        uses: Swatinem/rust-cache@v2
       - name: build
         shell: bash
         env:
@@ -161,7 +150,7 @@ jobs:
           path: .github/resources/config-for-runtime-upgrade-test.json
   # THE FOLLOWING HAPPENS ONLY WHEN TAGGED WITH A vX.Y.Z TAG
   create-draft-release:
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     needs: [build-runtimes]
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}
@@ -178,7 +167,7 @@ jobs:
           release_name: ${{ github.ref }}
           draft: true
   publish-runtimes:
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     needs: create-draft-release
     strategy:
       matrix:
@@ -243,7 +232,7 @@ jobs:
           asset_name: ${{ matrix.runtime.name }}-srtool-output-v${{ steps.get-runtime-version.outputs.runtime_ver }}.json
           asset_content_type: application/json
   publish-node:
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     needs: [build-node-current, create-draft-release]
     outputs:
       download_url: ${{ steps.upload-manta.outputs.browser_download_url }}
@@ -263,7 +252,7 @@ jobs:
           asset_name: manta
           asset_content_type: application/octet-stream
   brag:
-    runs-on: runtime
+    runs-on: ubuntu-20.04
     needs: [publish-runtimes, publish-node]
     if: startsWith(github.ref, 'refs/tags')
     steps: