bittensor-core e2es

.github/workflows/check-bittensor-e2e-tests.yml

@@ -50,7 +50,9 @@ on:
 env:
   CARGO_TERM_COLOR: always
   VERBOSE: ${{ github.event.inputs.verbose }}
-  # The in-repo SDK e2e harness reads LOCALNET_IMAGE from the environment.
+  # The Rust e2e harness prefers LOCALNET_IMAGE_NAME; Python fixtures still read
+  # LOCALNET_IMAGE, so keep both stable aliases pointed at the tested image.
+  LOCALNET_IMAGE_NAME: ghcr.io/raofoundation/subtensor-localnet:ci
   LOCALNET_IMAGE: ghcr.io/raofoundation/subtensor-localnet:ci
   # Per-commit tag on GHCR: built once, pulled by every test job. Registry
   # pulls beat the old artifact-tarball + docker-load path (~1-2 min/job)
@@ -119,59 +121,86 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-        with:
-          enable-cache: "true"
-          cache-dependency-glob: "sdk/python/uv.lock"
-
-      # One matrix entry per test FILE, not per test: every job pays ~2-3
-      # minutes of fixed cost (checkout, uv sync, downloading + loading the
-      # localnet image, booting a chain), so per-test fan-out spent ~20x more
-      # time on setup than on tests. The e2e conftest's fixtures are already
-      # session-scoped — one localnet shared by all tests in a pytest run —
-      # so a per-file job is the granularity the suite was written for.
-      - name: Collect in-repo SDK e2e tests
+      - name: Verify the Rust e2e migration manifest
         id: get-tests
-        working-directory: sdk/python
+        shell: bash
         run: |
           set -euo pipefail
-          uv sync --locked --all-extras --dev
-          test_matrix=$(
-            uv run pytest -q --collect-only -m e2e \
-            | sed -n '/^tests\/e2e\//p' \
-            | jq -R -s -c '
-                split("\n")
-                | map(select(. != ""))
-                | map(split("::")[0])
-                | unique
-                | map({nodeid: ., label: (sub("^tests/e2e/"; "") | sub("\\.py$"; ""))})
-              '
-          )
-          echo "Found test files: $test_matrix"
-          if [ "$test_matrix" = "[]" ] || [ -z "$test_matrix" ]; then
-            echo "No e2e tests found — failing."
-            exit 1
-          fi
+          manifest=sdk/bittensor-core/tests/e2e-manifest.json
+          jq -e 'length == 113' "$manifest" >/dev/null
+          jq -e 'map(.label) | length == (unique | length)' "$manifest" >/dev/null
+          jq -e 'map(.test) | length == (unique | length)' "$manifest" >/dev/null
+          jq -e 'all(.[]; (.label | startswith("test_")) and ((.test | startswith("intent_")) or (.test | startswith("test_"))))' "$manifest" >/dev/null
+          test_matrix=$(jq -c . "$manifest")
+          echo "Found $(jq length "$manifest") migrated Rust e2e tests"
           echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT"
+
+  build-rust-e2e-test-binary:
+    needs: [check-label, find-e2e-tests]
+    if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+
+      - name: Install Rust + build dependencies
+        run: |
+          chmod +x ./scripts/install_build_env.sh
+          ./scripts/install_build_env.sh
+
+      - name: Compile once and prove manifest completeness
         shell: bash
+        run: |
+          set -euo pipefail
+          source "$HOME/.cargo/env"
+          mkdir -p build/rust-e2e
+          # The workspace/localnet build also compiles the Python binding. Keep
+          # CoreError/API compatibility in this faster job so failures surface
+          # before the expensive node builds.
+          cargo check -p bittensor-core-py
+          cargo test -p bittensor-core --test e2e --no-run --message-format=json \
+            > build/rust-e2e/cargo-messages.json
+          executable=$(
+            jq -r '
+              select(.reason == "compiler-artifact")
+              | select(.target.name == "e2e")
+              | select(.target.kind | index("test"))
+              | .executable // empty
+            ' build/rust-e2e/cargo-messages.json | tail -n 1
+          )
+          test -n "$executable"
+          cp "$executable" build/rust-e2e/bittensor-core-e2e
+          chmod +x build/rust-e2e/bittensor-core-e2e
+
+          jq -r '.[].test' sdk/bittensor-core/tests/e2e-manifest.json \
+            | sort > build/rust-e2e/expected-tests.txt
+          build/rust-e2e/bittensor-core-e2e --list --format terse \
+            | sed -n 's/: test$//p' \
+            | sort > build/rust-e2e/compiled-tests.txt
+          diff -u build/rust-e2e/expected-tests.txt build/rust-e2e/compiled-tests.txt
+
+      - name: Upload compiled Rust e2e binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: bittensor-core-e2e
+          path: build/rust-e2e/bittensor-core-e2e
+          if-no-files-found: error
 
   artifacts:
-    name: Node • ${{ matrix.runtime }} • ${{ matrix.platform.arch }}
+    name: Node • fast-runtime • ${{ matrix.platform.arch }}
     needs: [check-label, trusted-pr]
     if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
     strategy:
+      fail-fast: false
       matrix:
         platform:
           - runner: [self-hosted, fireactions-heavy]
             triple: x86_64-unknown-linux-gnu
             arch: amd64
-        runtime: ["fast-runtime", "non-fast-runtime"]
 
     runs-on: ${{ matrix.platform.runner }}
 
@@ -202,15 +231,13 @@ jobs:
           export PATH="$HOME/.cargo/bin:$PATH"
           export CARGO_BUILD_TARGET="${{ matrix.platform.triple }}"
 
-          if [ "${{ matrix.runtime }}" = "fast-runtime" ]; then
-            ./scripts/localnet.sh --build-only
-          else
-            ./scripts/localnet.sh False --build-only
-          fi
+          # Historical Python E2Es used Docker's default `True`, so this
+          # migration intentionally validates the same fast-runtime localnet.
+          ./scripts/localnet.sh --build-only
 
       - name: Prepare artifacts for upload
         run: |
-          RUNTIME="${{ matrix.runtime }}"
+          RUNTIME="fast-runtime"
           TRIPLE="${{ matrix.platform.triple }}"
 
           BINARY_PATH="target/${RUNTIME}/${TRIPLE}/release/node-subtensor"
@@ -237,7 +264,7 @@ jobs:
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: binaries-${{ matrix.platform.triple }}-${{ matrix.runtime }}
+          name: binaries-${{ matrix.platform.triple }}-fast-runtime
           path: build/
           if-no-files-found: error
 
@@ -270,7 +297,7 @@ jobs:
           docker info | grep "Docker Root Dir"
 
       - name: Build Docker Image
-        run: docker build -f Dockerfile-localnet --build-arg BUILT_IN_CI="Boom shakalaka" -t "$LOCALNET_IMAGE_CI" .
+        run: docker build -f sdk/bittensor-core/tests/Dockerfile.localnet-fast -t "$LOCALNET_IMAGE_CI" .
 
       - name: Login to GHCR
         uses: docker/login-action@v3
@@ -286,6 +313,7 @@ jobs:
     needs:
       - check-label
       - find-e2e-tests
+      - build-rust-e2e-test-binary
       - build-image-with-current-branch
     if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
     runs-on: ubuntu-latest
@@ -296,24 +324,13 @@ jobs:
         include: ${{ fromJson(needs.find-e2e-tests.outputs.test-files) }}
 
     timeout-minutes: 60
-    name: "sdk: ${{ matrix.label }}"
+    name: "sdk-rust: ${{ matrix.label }}"
     steps:
-      - name: Check-out repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
-          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
+      - name: Download compiled Rust e2e binary
+        uses: actions/download-artifact@v4
         with:
-          enable-cache: "true"
-          cache-dependency-glob: "sdk/python/uv.lock"
-
-      - name: Install in-repo SDK
-        working-directory: sdk/python
-        run: |
-          uv sync --locked --all-extras --dev
+          name: bittensor-core-e2e
+          path: rust-e2e
 
       - name: Login to GHCR
         uses: docker/login-action@v3
@@ -326,34 +343,33 @@ jobs:
         run: docker pull "$LOCALNET_IMAGE_CI"
 
       - name: Retag Docker Image
-        run: docker tag "$LOCALNET_IMAGE_CI" "$LOCALNET_IMAGE"
+        run: |
+          docker tag "$LOCALNET_IMAGE_CI" "$LOCALNET_IMAGE_NAME"
+          docker tag "$LOCALNET_IMAGE_CI" "$LOCALNET_IMAGE"
 
       - name: Run with retry
-        working-directory: sdk/python
         env:
           SKIP_PULL: 1
+          RUST_BACKTRACE: 1
+          TEST_NAME: ${{ matrix.test }}
         run: |
           set +e
+          chmod +x rust-e2e/bittensor-core-e2e
           for i in 1 2; do
-            echo "Attempt $i: Running tests"
-            # -m e2e overrides the default addopts (-m 'not e2e') which would
-            # otherwise deselect the collected tests and fail with "no tests
-            # ran". The retry reruns only what failed (--lf reads pytest's
-            # cache from attempt 1) against a fresh localnet — the session
-            # fixture tears the container down and boots a new one per run.
-            if [ "$i" = "1" ]; then
-              uv run pytest "${{ matrix.nodeid }}" -s -m e2e
-            else
-              uv run pytest "${{ matrix.nodeid }}" -s -m e2e --lf
-            fi
+            echo "Attempt $i: Running $TEST_NAME"
+            rust-e2e/bittensor-core-e2e \
+              "$TEST_NAME" \
+              --exact \
+              --nocapture \
+              --test-threads=1
             status=$?
             if [ $status -eq 0 ]; then
-              echo "Tests passed on attempt $i"
+              echo "Test passed on attempt $i"
               break
             else
-              echo "Tests failed on attempt $i"
+              echo "Test failed on attempt $i"
               if [ $i -eq 2 ]; then
-                echo "Tests failed after 2 attempts"
+                echo "Test failed after 2 attempts"
                 exit 1
               fi
               echo "Retrying..."

sdk/bittensor-core-py/src/errors.rs

@@ -1,7 +1,7 @@
 //! The one place `CoreError` becomes a Python exception.
 
 use bittensor_core::CoreError;
-use pyo3::exceptions::{PyKeyError, PyValueError};
+use pyo3::exceptions::{PyConnectionError, PyKeyError, PyPermissionError, PyValueError};
 use pyo3::prelude::*;
 
 pyo3::create_exception!(
@@ -33,5 +33,7 @@ pub fn to_py_err(error: CoreError) -> PyErr {
         CoreError::NotInRuntime(what) => PyKeyError::new_err(what),
         CoreError::Codec(msg) | CoreError::Crypto(msg) => PyValueError::new_err(msg),
         CoreError::Device(msg) => LedgerError::new_err(msg),
+        CoreError::Rpc(msg) => PyConnectionError::new_err(msg),
+        CoreError::Policy(msg) => PyPermissionError::new_err(msg),
     }
 }

sdk/bittensor-core/Cargo.toml

@@ -10,6 +10,13 @@ license = "Apache-2.0"
 name = "bittensor_core"
 crate-type = ["rlib"]
 
+# External localnet suite. It is selected explicitly by the dedicated E2E
+# workflow; `test = false` keeps ordinary workspace/nextest runs unit-only.
+[[test]]
+name = "e2e"
+path = "tests/e2e.rs"
+test = false
+
 [dependencies]
 # keys / keyfiles (absorbed from py-sp-core). The optional crates below are
 # host-only (see the `host` feature): they either do I/O or compile C, both
@@ -68,6 +75,7 @@ merkleized-metadata = "0.5.1"
 # workers (codec/batch.rs).
 rayon = { version = "1.10", optional = true }
 scale-info = { workspace = true, features = ["std", "serde"] }
+subtensor-macros = { workspace = true }
 
 # signers (feature "ledger"): APDU types for the Polkadot generic app; the
 # HID transport itself lives in signers/ledger.rs on top of hidapi (below).