Auto merge of #123451 - Kobzol:arbitrary-try-build, r=pietroalbini

CI: add a script for dynamically computing CI job matrix

It would be great if was easier to run specific CI workflows locally, and also to allow us to spawn a specific CI workflow by bors, to enable running arbitrary try builds. See discussion [here](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/CI.20workflows.20refactoring).

This PR is a first step in that direction.
- Moves the definition of CI runners and (for now) PR jobs into a separate `jobs.yml` file.
- Adds a simple Python script that reads the file, decides which jobs should be active for the current CI workflow, and prints them as JSON to their output.
- The PR job then reads this output and generates its job matrix based on it.

By moving the job definitions from `ci.yml` into a separate file, we can handle it programmatically, which should make it easier to both do local execution of CI jobs and also to do arbitrary try builds.

Author: bors

.github/workflows/ci.yml

@@ -36,8 +36,21 @@ concurrency:
   group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
   cancel-in-progress: true
 jobs:
+  calculate_matrix:
+    name: Calculate job matrix
+    runs-on: ubuntu-latest
+    outputs:
+      jobs: "${{ steps.jobs.outputs.jobs }}"
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+      - name: Calculate the CI job matrix
+        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        id: jobs
   pr:
     name: "PR - ${{ matrix.name }}"
+    needs:
+      - calculate_matrix
     env:
       PR_CI_JOB: 1
       CI_JOB_NAME: "${{ matrix.name }}"
@@ -51,20 +64,7 @@ jobs:
     continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
     strategy:
       matrix:
-        include:
-          - name: mingw-check
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: mingw-check-tidy
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: x86_64-gnu-llvm-17
-            env:
-              ENABLE_GCC_CODEGEN: "1"
-            os: ubuntu-20.04-16core-64gb
-          - name: x86_64-gnu-tools
-            os: ubuntu-20.04-16core-64gb
-            env: {}
+        include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
     defaults:
       run:
         shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"

src/ci/github-actions/ci.yml

@@ -341,30 +341,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
+  calculate_matrix:
+    name: Calculate job matrix
+    runs-on: ubuntu-latest
+    outputs:
+      jobs: ${{ steps.jobs.outputs.jobs }}
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+      - name: Calculate the CI job matrix
+        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        id: jobs
   pr:
     <<: *base-ci-job
     name: PR - ${{ matrix.name }}
+    needs: [ calculate_matrix ]
     env:
       <<: [*shared-ci-variables, *public-variables]
       PR_CI_JOB: 1
     if: github.event_name == 'pull_request'
     continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
     strategy:
       matrix:
-        include:
-          - name: mingw-check
-            <<: *job-linux-4c
-
-          - name: mingw-check-tidy
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-llvm-17
-            env:
-              ENABLE_GCC_CODEGEN: "1"
-            <<: *job-linux-16c
-
-          - name: x86_64-gnu-tools
-            <<: *job-linux-16c
+        # Check the `calculate_matrix` job to see how is the matrix defined.
+        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
 
   auto:
     <<: *base-ci-job

src/ci/github-actions/jobs.yml

@@ -0,0 +1,50 @@
+# This file contains definitions of CI job parameters that are loaded
+# dynamically in CI from ci.yml.
+# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
+# modify this file.
+shared_defs:
+  - &base-job
+    env: { }
+
+  - &job-linux-4c
+    os: ubuntu-20.04-4core-16gb
+    <<: *base-job
+
+  - &job-linux-8c
+    os: ubuntu-20.04-8core-32gb
+    <<: *base-job
+
+  - &job-linux-16c
+    os: ubuntu-20.04-16core-64gb
+    <<: *base-job
+
+  - &job-macos-xl
+    os: macos-13 # We use the standard runner for now
+    <<: *base-job
+
+  - &job-macos-m1
+    os: macos-14
+    <<: *base-job
+
+  - &job-windows-8c
+    os: windows-2019-8core-32gb
+    <<: *base-job
+
+  - &job-windows-16c
+    os: windows-2019-16core-64gb
+    <<: *base-job
+
+  - &job-aarch64-linux
+    os: [ self-hosted, ARM64, linux ]
+
+pr:
+  - name: mingw-check
+    <<: *job-linux-4c
+  - name: mingw-check-tidy
+    <<: *job-linux-4c
+  - name: x86_64-gnu-llvm-17
+    env:
+      ENABLE_GCC_CODEGEN: "1"
+    <<: *job-linux-16c
+  - name: x86_64-gnu-tools
+    <<: *job-linux-16c

src/ci/scripts/calculate-job-matrix.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+"""
+This script serves for generating a matrix of jobs that should
+be executed on CI.
+
+It reads job definitions from `src/ci/github-actions/jobs.yml`
+and filters them based on the event that happened on CI.
+
+Currently, it only supports PR builds.
+"""
+
+import json
+from pathlib import Path
+
+import yaml
+
+JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml"
+
+
+if __name__ == "__main__":
+    with open(JOBS_YAML_PATH) as f:
+        jobs = yaml.safe_load(f)
+    job_output = jobs["pr"]
+    print(f"jobs={json.dumps(job_output)}")

src/tools/expand-yaml-anchors/src/main.rs

@@ -2,11 +2,11 @@ use std::error::Error;
 use std::path::{Path, PathBuf};
 use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
 
-/// List of directories containing files to expand. The first tuple element is the source
-/// directory, while the second tuple element is the destination directory.
+/// List of files to expand. The first tuple element is the source
+/// file, while the second tuple element is the destination file.
 #[rustfmt::skip]
 static TO_EXPAND: &[(&str, &str)] = &[
-    ("src/ci/github-actions", ".github/workflows"),
+    ("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"),
 ];
 
 /// Name of a special key that will be removed from all the maps in expanded configuration files.
@@ -62,27 +62,20 @@ impl App {
     fn run(&self) -> Result<(), Box<dyn Error>> {
         for (source, dest) in TO_EXPAND {
             let source = self.base.join(source);
-            let dest = self.base.join(dest);
-            for entry in std::fs::read_dir(&source)? {
-                let path = entry?.path();
-                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
-                    continue;
-                }
-
-                let dest_path = dest.join(path.file_name().unwrap());
-                self.expand(&path, &dest_path).with_context(|| match self.mode {
-                    Mode::Generate => format!(
-                        "failed to expand {} into {}",
-                        self.path(&path),
-                        self.path(&dest_path)
-                    ),
-                    Mode::Check => format!(
-                        "{} is not up to date; please run \
+            let dest_path = self.base.join(dest);
+
+            self.expand(&source, &dest_path).with_context(|| match self.mode {
+                Mode::Generate => format!(
+                    "failed to expand {} into {}",
+                    self.path(&source),
+                    self.path(&dest_path)
+                ),
+                Mode::Check => format!(
+                    "{} is not up to date; please run \
                         `x.py run src/tools/expand-yaml-anchors`.",
-                        self.path(&dest_path)
-                    ),
-                })?;
-            }
+                    self.path(&dest_path)
+                ),
+            })?;
         }
         Ok(())
     }