ci: Build statically linked bpf-linker in CI

vadorovsky · vadorovsky · commit 35354c9af297 · 2024-08-18T07:42:59.000+02:00
- Download LLVM builds from Rust CI.
- Always link libLLVM statically.
- For now, do it only for x86_64.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -49,17 +49,18 @@ jobs:
 
   build:
     runs-on: ubuntu-22.04
+    container:
+      image: alpine:3.20
     strategy:
       fail-fast: false
       matrix:
         rust:
           - stable
           - beta
           - nightly
-        llvm:
-          - 19
-          - source
-    name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }}
+        target:
+          - x86_64-unknown-linux-musl
+    name: rustc=${{ matrix.rust }} target=${{ matrix.target }}
     needs: llvm
 
     env:
@@ -68,6 +69,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install dependencies
+        run: |
+          set -euxo pipefail
+          apk update
+          apk add bash curl libgcc gcc
+
       - name: Install Rust ${{ matrix.rust }}
         if: matrix.rust != 'nightly'
         uses: dtolnay/rust-toolchain@master
@@ -81,7 +88,7 @@ jobs:
           toolchain: ${{ matrix.rust }}
           components: rust-src
           # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
-          targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl
+          targets: ${{ matrix.target }}-unknown-linux-musl
 
       - uses: Swatinem/rust-cache@v2
 
@@ -91,59 +98,33 @@ jobs:
       - name: Build (default features, no system LLVM)
         run: cargo build
 
-      - name: Install dependencies
-        if: matrix.rust == 'nightly'
-        # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
-        # enum values which was added in clang 15[1].
-        #
-        # gcc-multilib provides at least <asm/types.h> which is referenced by libbpf.
-        #
-        # llvm provides llvm-objcopy which is used to build the BTF relocation tests.
-        #
-        # [0] https://github.com/actions/runner-images/blob/ubuntu22/20230724.1/images/linux/Ubuntu2204-Readme.md
-        #
-        # [1] https://github.com/llvm/llvm-project/commit/dc1c43d
-        run: |
-          set -euxo pipefail
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt update
-          sudo apt -y install clang gcc-multilib
-
-      - name: Install LLVM
-        if: matrix.llvm != 'source'
-        run: |
-          set -euxo pipefail
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list
-
-          sudo apt update
-          # TODO(vadorovsky): Remove the requirement of libpolly.
-          #
-          # Packages from apt.llvm.org are being built all at once, with one
-          # cmake build with superset of options, then different binaries and
-          # libraries are being included in different packages.
-          #
-          # That results in `llvm-config --libname --link-static` mentioning
-          # libpolly, even if it's not installed. The output of that command is
-          # being used in build.rs of llvm-sys, so building llvm-sys on such
-          # system is complaining about lack of libpolly.
-          #
-          # Hopefully that nightmare goes away once we switch to binstalls and
-          # ditch the system LLVM option.
-          sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev
-          echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH
+      # - name: Install dependencies
+      #   if: matrix.rust == 'nightly'
+      #   # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
+      #   # enum values which was added in clang 15[1].
+      #   #
+      #   # gcc-multilib provides at least <asm/types.h> which is referenced by libbpf.
+      #   #
+      #   # llvm provides llvm-objcopy which is used to build the BTF relocation tests.
+      #   #
+      #   # [0] https://github.com/actions/runner-images/blob/ubuntu22/20230724.1/images/linux/Ubuntu2204-Readme.md
+      #   #
+      #   # [1] https://github.com/llvm/llvm-project/commit/dc1c43d
+      #   run: |
+      #     set -euxo pipefail
+      #     wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+      #     echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
+      #     sudo apt update
+      #     sudo apt -y install clang gcc-multilib
 
       - name: Restore LLVM
-        if: matrix.llvm == 'source'
         uses: actions/cache/restore@v4
         with:
           path: llvm-install
           key: ${{ needs.llvm.outputs.cache-key }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH && LD_LIBRARY_PATH
-        if: matrix.llvm == 'source'
         run: |
           set -euxo pipefail
           echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH
@@ -220,3 +201,7 @@ jobs:
           set -euxo pipefail
           sudo apt install -y locate qemu-system-{arm,x86}
           find test/.tmp -name 'vmlinuz-*' | RUSTFLAGS=-Cdebuginfo=line-directives-only xargs -t cargo xtask integration-test vm
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml
@@ -9,18 +9,38 @@ on:
 jobs:
   llvm:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-musl
     name: llvm
     outputs:
       cache-key: ${{ steps.cache-key.outputs.cache-key }}
     steps:
+      - name: Install Rust nightly
+        uses: dtolnay/rust-toolchain@nightly
+
+      # We retrieve the commit hashes of two repositories:
+      #
+      # - rust-lang/rust - the Rust monorepo. Rust CI is using commit hashes
+      #   from the monorepo for annotating LLVM tarballs. We use it to download
+      #   the tarball.
+      # - aya-rs/llvm-project - which is our fork of LLVM, kept in sync with
+      #   rust-lang/llvm-project. We use it in case we can't download LLVM
+      #   toolchain from Rust CI.
       - id: ls-remote
         run: |
           set -euxo pipefail
-          value=$(git ls-remote https://github.com/aya-rs/llvm-project.git refs/heads/rustc/19.1-2024-07-30 | cut -f1)
-          echo "sha=$value" >> "$GITHUB_OUTPUT"
+          # Get the partial SHA from Rust nightly.
+          rustc_sha=$(rustc +nightly --version | grep -oE '[a-f0-9]{7,40}')
+          # Get the full SHA from GitHub.
+          rustc_sha=$(curl -s https://api.github.com/repos/rust-lang/rust/commits/$rustc_sha | jq -r '.sha')
+          aya_llvm_sha=$(git ls-remote https://github.com/aya-rs/llvm-project.git refs/heads/rustc/19.1-2024-07-30 | cut -f1)
+          echo "rustc-sha=$rustc_sha" >> "$GITHUB_OUTPUT"
+          echo "aya-llvm-sha=$aya_llvm_sha" >> "$GITHUB_OUTPUT"
 
       - id: cache-key
-        run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
+        run: echo "cache-key=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.rustc-sha }}-1" >> "$GITHUB_OUTPUT"
 
       - name: Cache
         id: cache-llvm
@@ -30,8 +50,17 @@ jobs:
           key: ${{ steps.cache-key.outputs.cache-key }}
           lookup-only: true
 
-      - name: Install Tools
+      - name: Download LLVM from Rust CI
+        id: download-llvm
         if: steps.cache-llvm.outputs.cache-hit != 'true'
+        run: |
+          set -euxo pipefail
+          mkdir -p llvm-install
+          wget -q -O - https://ci-artifacts.rust-lang.org/rustc-builds/${{ steps.ls-remote.outputs.rustc-sha }}/rust-dev-nightly-${{ matrix.target }}.tar.xz | \
+            tar -xJ --strip-components 2 -C llvm-install
+
+      - name: Install Tools
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && steps.download-llvm.outcome != 'success'
         run: |
           set -euxo pipefail
           wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
@@ -44,15 +73,15 @@ jobs:
           sudo apt -y install cmake ninja-build clang lld
 
       - name: Checkout LLVM Source
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && steps.download-llvm.outcome != 'success'
         uses: actions/checkout@v4
         with:
           repository: aya-rs/llvm-project
-          ref: ${{ steps.ls-remote.outputs.sha }}
+          ref: ${{ steps.ls-remote.outputs.aya-llvm-sha }}
           path: llvm-project
 
       - name: Configure LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && steps.download-llvm.outcome != 'success'
         run: |
           set -euxo pipefail
           cmake \
@@ -73,7 +102,7 @@ jobs:
             -DLLVM_USE_LINKER=lld
 
       - name: Install LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && steps.download-llvm.outcome != 'success'
         env:
           # Create symlinks rather than copies to conserve disk space. At the time of this writing,
           # GitHub-hosted runners have 14GB of SSD space
@@ -85,7 +114,7 @@ jobs:
         run: cmake --build llvm-build --target install
 
       - name: Rewrite LLVM Symlinks
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && steps.download-llvm.outcome != 'success'
         # Move targets over the symlinks that point to them.
         #
         # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE.
