ci: cache downloads

We're seeing 429 from Github trying to download gen_init_cpio, so cache
it using actions and curl's ETag support. Since I'm here add this for
kernel images as well to save time waiting on slow Debian servers.

Author: tamird

.github/scripts/download_kernel_images.sh

@@ -29,5 +29,24 @@ for VERSION in "${VERSIONS[@]}"; do
   )"
 done
 
-printf '%s\n' "${FILES[@]}" \
-| xargs -t curl -sfSL --create-dirs --output-dir "$OUTPUT_DIR" --parallel --remote-name-all
+# TODO(https://github.com/curl/curl/issues/15729): restore --parallel here if and when it properly
+# supports ETags.
+# TODO(https://github.com/curl/curl/issues/15730): restore --create-dirs when it works in the
+# presence of `--etag-save`.`
+mkdir -p "$OUTPUT_DIR"
+KEEP=()
+for FILE in "${FILES[@]}"; do
+  name=$(basename "$FILE")
+  etag_name="$name.etag"
+  KEEP+=("$name" "$etag_name")
+
+  etag="$OUTPUT_DIR/$etag_name"
+  curl -sfSL --output-dir "$OUTPUT_DIR" --remote-name-all --etag-compare "$etag" --etag-save "$etag" "$FILE"
+done
+
+# Remove any files that were previously downloaded that are no longer needed.
+FIND_ARGS=()
+for FILE in "${KEEP[@]}"; do
+  FIND_ARGS+=("!" "-name" "$FILE")
+done
+find "$OUTPUT_DIR" -type f "${FIND_ARGS[@]}" -exec rm {} +

.github/workflows/ci.yml

@@ -226,7 +226,9 @@ jobs:
 
       - name: Install prerequisites
         if: runner.os == 'macOS'
-        # The xargs shipped on macOS always exits 0 with -P0, so we need GNU findutils.
+        # The curl shipped on macOS doesn't contain
+        # https://github.com/curl/curl/commit/85efbb92b8e6679705e122cee45ce76c56414a3e which is
+        # needed for proper handling of `--etag-{compare,save}`.
         #
         # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar.
         #
@@ -237,9 +239,10 @@ jobs:
           # https://github.com/actions/setup-python/issues/577
           find /usr/local/bin -type l -exec sh -c 'readlink -f "$1" \
           | grep -q ^/Library/Frameworks/Python.framework/Versions/' _ {} \; -exec rm -v {} \;
-          brew install --formula dpkg gnu-tar llvm lynx pkg-config qemu
-          echo $(brew --prefix)/opt/gnu-tar/libexec/gnubin >> $GITHUB_PATH
-          echo $(brew --prefix)/opt/llvm/bin >> $GITHUB_PATH
+          brew install --formula curl dpkg gnu-tar llvm lynx pkg-config qemu
+          echo $(brew --prefix curl)/bin >> $GITHUB_PATH
+          echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH
+          echo $(brew --prefix llvm)/bin >> $GITHUB_PATH
 
       - uses: dtolnay/rust-toolchain@nightly
         with:
@@ -278,6 +281,12 @@ jobs:
         # Do this on all system (not just macOS) to avoid relying on rustc-provided libLLVM.so.
         run: cargo install --force bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features
 
+      - name: Cache test cache
+        uses: actions/cache@v4
+        with:
+          path: test/.tmp
+          key: ${{ runner.arch }}-test-cache
+
       - name: Download debian kernels
         if: runner.arch == 'ARM64'
         # TODO: enable tests on kernels before 6.0.
@@ -299,7 +308,10 @@ jobs:
         run: cargo xtask integration-test local
 
       - name: Run virtualized integration tests
-        run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm
+        run: |
+          set -euxo pipefail
+          find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
+            cargo xtask integration-test vm --cache-dir test/.tmp
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

test/README.md

@@ -27,7 +27,7 @@ cargo xtask integration-test local
 ### Virtualized
 
 ```bash
-cargo xtask integration-test vm <KERNEL IMAGE>
+cargo xtask integration-test vm --cache-dir <CACHE_DIR> <KERNEL_IMAGE>...
 ```
 
 ### Writing an integration test

xtask/src/run.rs

@@ -3,7 +3,7 @@ use std::{
     fmt::Write as _,
     fs::{copy, create_dir_all, OpenOptions},
     io::{BufRead as _, BufReader, Write as _},
-    path::{Path, PathBuf},
+    path::PathBuf,
     process::{Child, ChildStdin, Command, Output, Stdio},
     sync::{Arc, Mutex},
     thread,
@@ -38,6 +38,10 @@ enum Environment {
         ///   | tar --wildcards --extract '*vmlinuz*' --file -"
         #[clap(required = true)]
         kernel_image: Vec<PathBuf>,
+
+        /// The cache directory in which to store utilities fetched from the internet.
+        #[clap(short, long)]
+        cache_dir: PathBuf,
     },
 }
 
@@ -167,7 +171,10 @@ pub fn run(opts: Options) -> Result<()> {
                 Err(anyhow!("failures:\n{}", failures))
             }
         }
-        Environment::VM { kernel_image } => {
+        Environment::VM {
+            kernel_image,
+            cache_dir,
+        } => {
             // The user has asked us to run the tests on a VM. This is involved; strap in.
             //
             // We need tools to build the initramfs; we use gen_init_cpio from the Linux repository,
@@ -185,20 +192,26 @@ pub fn run(opts: Options) -> Result<()> {
             // We consume the output of QEMU, looking for the output of our init program. This is
             // the only way to distinguish success from failure. We batch up the errors across all
             // VM images and report to the user. The end.
-            let cache_dir = Path::new("test/.tmp");
-            create_dir_all(cache_dir).context("failed to create cache dir")?;
+            create_dir_all(&cache_dir).context("failed to create cache dir")?;
             let gen_init_cpio = cache_dir.join("gen_init_cpio");
             if !gen_init_cpio
                 .try_exists()
                 .context("failed to check existence of gen_init_cpio")?
             {
+                let gen_init_cpio_etag = cache_dir.join("gen_init_cpio.etag");
                 let mut curl = Command::new("curl");
+                for flag in ["--etag-compare", "--etag-save"] {
+                    curl.arg(flag).arg(&gen_init_cpio_etag);
+                }
                 curl.args([
                     "-sfSL",
                     "https://raw.githubusercontent.com/torvalds/linux/master/usr/gen_init_cpio.c",
                 ]);
                 let mut curl_child = curl
+                    // Sent to clang.
                     .stdout(Stdio::piped())
+                    // Collected for error reporting.
+                    .stderr(Stdio::piped())
                     .spawn()
                     .with_context(|| format!("failed to spawn {curl:?}"))?;
                 let Child { stdout, .. } = &mut curl_child;