chore(ci): Move subrepo e2e tests to workflows (#675)

Meaningfully faster; more product usage. Includes pre-commit automation
for the lazy (me). Need to finish the bzlmod migration to re-enable bzl8.

### Changes are visible to end-users: no

### Test plan

This is the test plan

---------

Co-authored-by: aspect-marvin[bot] <[email protected]>

Author: arrdem

.aspect/workflows/config.yaml

@@ -1,8 +1,63 @@
 # See https://docs.aspect.build/workflows/configuration
+
+# This list is maintained by tools/e2e/sync.py
+workspaces:
+  # SYNC: e2e
+  # {{{
+  - .
+  - e2e/cross-repo-610
+  - e2e/interpreter-version-541
+  - e2e/repository-rule-deps
+  - e2e/smoke
+  - e2e/system-interpreter
+  - examples/uv_pip_compile
+# }}}
+
 tasks:
+  # TODO(arrdem): Rosetta doesn't like HEAD version strings seemingly
+  # - test:
+  #     name: test-os:linux-bzl:head
+  #     queue: aspect-default
+  #     hooks:
+  #       - type: before_task
+  #         command: |
+  #           rm .bazelversion
+  #           echo "USE_BAZEL_VERSION=last_green" > .bazeliskrc
+
+  # TODO(arrdem): Bazel 8+ dropped the original PyInfo etc. which the ancient
+  # rules_python we have on the WORKSPACE side isn't ready for yet. Once we kill
+  # off WORKSPACE we can flip these back on but practically this is blocked on
+  # getting that change in.
+  #
+  # - test:
+  #     name: test-os:linux-bzl:latest
+  #     queue: aspect-large
+  #     hooks:
+  #       - type: before_task
+  #         command: |
+  #           rm .bazelversion
+  #           echo "USE_BAZEL_VERSION=latest" > .bazeliskrc
+
+  # - test:
+  #     name: test-os:linux-bzl:8
+  #     queue: aspect-large
+  #     hooks:
+  #       - type: before_task
+  #         command: |
+  #           rm .bazelversion
+  #           echo "USE_BAZEL_VERSION=8.2.1" > .bazeliskrc
+
   - test:
+      name: test-os:linux-bzl:7
       queue: aspect-large
+      hooks:
+        - type: before_task
+          command: |
+            rm .bazelversion
+            echo "USE_BAZEL_VERSION=7.6.1" > .bazeliskrc
+
   - finalization:
       queue: aspect-small
+
 notifications:
   github: {}

.bazelrc

@@ -6,6 +6,8 @@ test --test_output=errors
 
 # TODO(alex): enable
 common --noenable_bzlmod
+# For very recent bazel we have to set this explicitly
+common --enable_workspace
 
 # Define value used by tests
 build --define=SOME_VAR=SOME_VALUE

.bazelversion

@@ -1,4 +1,4 @@
-7.4.0
+7.6.1
 # The first line of this file is used by Bazelisk and Bazel to be sure
 # the right version of Bazel is used to build and test this repo.
 # This also defines which version is used on CI.

.github/workflows/ci.yaml

@@ -22,33 +22,6 @@ concurrency:
   cancel-in-progress: ${{ github.ref_name != 'main' }}
 
 jobs:
-  test:
-    uses: bazel-contrib/.github/.github/workflows/bazel.yaml@646899086d7aaee8e532540480f3e91e00596234 # 11 Dec 2024
-    with:
-      # NB: the root folder is tested with Aspect Workflows on BuildKite, see /.aspect/workflows/config.yaml
-      folders: |
-        [
-          "e2e/cross-repo-610",
-          "e2e/interpreter-version-541",
-          "e2e/repository-rule-deps",
-          "e2e/smoke",
-          "e2e/system-interpreter",
-          "examples/uv_pip_compile"
-        ]
-      # TODO: Build Windows tools and add to toolchain
-      # TODO(alex): switch the root folder to bzlmod
-      # TODO: fix remaining folders on Bazel 8
-      exclude: |
-        [
-          {"os": "windows-latest"},
-          {"folder": "e2e/cross-repo-610", "bzlmodEnabled": false},
-          {"folder": "e2e/interpreter-version-541", "bzlmodEnabled": false},
-          {"folder": "e2e/repository-rule-deps", "bzlmodEnabled": false},
-          {"folder": "e2e/system-interpreter", "bzlmodEnabled": false},
-          {"folder": "examples/uv_pip_compile", "bzlmodEnabled": false},
-          {"folder": "e2e/smoke", "bazelversion": "8.0.0"}
-        ]
-
   pre-commit:
     runs-on: ubuntu-latest
     if: always() && !cancelled()
@@ -114,7 +87,6 @@ jobs:
   # on "all matrix jobs were successful".
   conclusion:
     needs:
-      - test
       - verify-bcr-patches
       - test-release
       - pre-commit

.pre-commit-config.yaml

@@ -61,3 +61,10 @@ repos:
         entry: /bin/sh -c '/usr/bin/env bazel run //py/tests/py_venv_image_layer:my_app_arm64_layers'
         language: script
         require_serial: true
+
+      - id: sync-workflows-subrepos
+        name: Update workflows file for subrepos
+        # Note that we use a nested shell to discard $@, which is the file list
+        entry: /usr/bin/env python3 tools/e2e/sync.py
+        language: script
+        require_serial: true

py/tests/py_image_layer/my_app_layers_listing.yaml

@@ -2491,6 +2491,6 @@ files:
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/bazel_tools/tools/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/
-  - -rwxr-xr-x  0 0      0       21622 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
+  - -rwxr-xr-x  0 0      0       21636 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.runfiles/pypi_colorama/
   - -rwxr-xr-x  0 0      0         183 Jan  1  2023 ./py/tests/py_image_layer/my_app_bin.venv.pth

py/tests/py_venv_image_layer/my_app_amd64_layers_listing.yaml

@@ -2523,5 +2523,5 @@ files:
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/
-  - -rwxr-xr-x  0 0      0       21622 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
+  - -rwxr-xr-x  0 0      0       21636 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/pypi_colorama/

py/tests/py_venv_image_layer/my_app_arm64_layers_listing.yaml

@@ -2504,5 +2504,5 @@ files:
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/
-  - -rwxr-xr-x  0 0      0       21622 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
+  - -rwxr-xr-x  0 0      0       21636 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash
   - drwxr-xr-x  0 0      0           0 Jan  1  2023 ./py/tests/py_venv_image_layer/my_app_bin.runfiles/pypi_colorama/

tools/e2e/BUILD.bazel

@@ -11,3 +11,9 @@ py_binary(
     srcs = ["devintegrity.py"],
     main = "devintegrity.py",
 )
+
+py_binary(
+    name = "sync",
+    srcs = ["sync.py"],
+    main = "sync.py",
+)

tools/e2e/sync.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import re
+from os import getenv
+from pathlib import Path
+
+WORKSPACES_BLOCK_RE = re.compile(r"""\
+# SYNC: e2e
+# {{{
+(.*?
+)# }}}
+""", re.MULTILINE | re.DOTALL)
+
+if __name__ == "__main__" or 1:
+    root = Path(__file__).parent.parent.parent.absolute()
+    workflows_file = root / ".aspect/workflows/config.yaml"
+    modules = root.rglob("**/MODULE.bazel")
+    module_roots = [it.parent.relative_to(root) for it in modules]
+
+    # Module roots that we know we can't run in workflows
+    ignored_roots = [
+        Path("e2e/use_release"),
+        Path("e2e/cross-repo-610/subrepo_a"),
+        Path("e2e/cross-repo-610/subrepo_b"),
+    ]
+
+    module_roots = [
+        r for r in module_roots if r not in ignored_roots
+    ]
+    module_roots = sorted(module_roots)
+    
+   
+    with open(workflows_file, "r") as fp:
+        config = fp.read()
+
+    def _handler(m):
+        return m.group(0).replace(m.group(1), "".join(["  - {}\n".format(it) for it in module_roots]))
+
+    new_config = re.sub(WORKSPACES_BLOCK_RE, _handler, config)
+
+    if config != new_config:
+        print(new_config)
+
+        with open(workflows_file, "w") as fp:
+            fp.write(new_config)
+
+        exit(1)