ci: fail closed when CodeQL analysis fails

.github/workflows/codeql.yml

@@ -38,6 +38,5 @@ jobs:
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v4.37.7
-        continue-on-error: true
         with:
           category: /language:python

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## Unreleased
 
+## v0.5.1 — 2026-08-22
+
+- Level 1 and Level 2 verification now requires a verifier-issued challenge via
+  `--expected-nonce` and checks it against the signed `runtime.nonce` using
+  constant-time comparison. Previously nonce binding existed only as an
+  assertion over the repository's own pytest fixture; the shipped runner and
+  CLI could report conformance for a fresh signed record containing an
+  attacker-chosen or replayed nonce.
+
 ## v0.5.0 — 2026-08-09
 
 ### Added

README.md

@@ -35,7 +35,8 @@ Seven test modules covering the full specification: envelope structure, signatur
 
 ```bash
 pip install agentrust-trace-tests
-trace-tests verify --record path/to/trust-record.jwt --level 1
+trace-tests verify --record path/to/trust-record.jwt --level 1 \
+  --expected-nonce "$VERIFIER_CHALLENGE"
 ```
 
 ## A report you can hand to someone else

docs/error-codes.md

@@ -28,6 +28,7 @@ All TRACE test failures emit a structured error code of the form `TR-<MODULE>-<N
 | TR-RTE-001 | `runtime` is missing or not an object, or `runtime.platform` is not in the registered set, or is `software-only` at Level 1 and above | Use a value from the `runtime.platform` enum in `schemas/trace-claim.json`. `software-only` carries no hardware attestation evidence and is accepted only at Level 0 |
 | TR-RTE-002 | `runtime.measurement` is not a valid `sha256:` digest | Provide a 64-character hex digest prefixed with `sha256:`; for Level 0 all-zeros is conventional |
 | TR-RTE-003 | `runtime.rim_uri` is present and is not an `https://` URI | Remove `runtime.rim_uri` if not using a RIM, or set it to an `https://` URI. The URI is not resolved and the manifest behind it is not checked; this is a format check |
+| TR-RTE-004 | Level 1+ verification is missing the verifier challenge nonce or the nonce does not match | Supply the verifier's expected nonce and require the attested runtime nonce to match it |
 
 ## TR-POL — Policy
 

docs/quickstart.md

@@ -81,8 +81,8 @@ Level 0 is software-only (development). Level 1 requires TEE attestation. Level
 
 ```bash
 trace-tests verify --record sample-record.json --level 0
-trace-tests verify --record sample-record.json --level 1
-trace-tests verify --record sample-record.json --level 2
+trace-tests verify --record sample-record.json --level 1 --expected-nonce "$VERIFIER_CHALLENGE"
+trace-tests verify --record sample-record.json --level 2 --expected-nonce "$VERIFIER_CHALLENGE"
 ```
 
 The sample fixture passes Level 0. Levels 1 and 2 will fail on runtime attestation and transparency fields — that is expected. See [Trust Levels](levels.md) for what each level requires.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentrust-trace-tests"
-version = "0.5.0"
+version = "0.5.1"
 description = "TRACE conformance test suite"
 readme = "README.md"
 license = { text = "Apache-2.0" }

src/trace_tests/cli.py

@@ -89,7 +89,13 @@ def main() -> None:
 
 @main.command()
 @click.option("--record", required=True, type=click.Path(), help="Path to the trust record (JSON)")
-@click.option("--level", default=0, type=click.IntRange(0, 2), show_default=True, help="Conformance level to check (0, 1, or 2)")
+@click.option(
+    "--level",
+    default=0,
+    type=click.IntRange(0, 2),
+    show_default=True,
+    help="Conformance level to check (0, 1, or 2)",
+)
 @click.option(
     "--max-age",
     "max_age",
@@ -98,15 +104,26 @@ def main() -> None:
     show_default=True,
     help="Maximum allowed record age in seconds (iat freshness window)",
 )
-def verify(record: str, level: int, max_age: int) -> None:
+@click.option(
+    "--expected-nonce",
+    default=None,
+    help="Verifier-issued challenge nonce; required for Level 1 and Level 2.",
+)
+def verify(record: str, level: int, max_age: int, expected_nonce: str | None) -> None:
     """Verify a TRACE trust record against the conformance suite."""
     try:
         data, fmt = load_record(record)
     except LoadError as exc:
         click.echo(f"Error: {exc}", err=True)
         sys.exit(2)
 
-    results = run(data, fmt, level, max_age_seconds=max_age)
+    results = run(
+        data,
+        fmt,
+        level,
+        max_age_seconds=max_age,
+        expected_nonce=expected_nonce,
+    )
     exit_code = _print_report(record, fmt, level, results)
     sys.exit(exit_code)
 
@@ -140,6 +157,11 @@ def verify(record: str, level: int, max_age: int) -> None:
     help="Exit non-zero unless the record reaches this level. Omit to always exit 0, "
     "which is what you want when generating an artifact rather than gating on one.",
 )
+@click.option(
+    "--expected-nonce",
+    default=None,
+    help="Verifier-issued challenge nonce; required for Level 1 and Level 2.",
+)
 def report(
     record: str,
     max_level: int,
@@ -148,6 +170,7 @@ def report(
     json_out: str | None,
     badge_out: str | None,
     fail_under: int | None,
+    expected_nonce: str | None,
 ) -> None:
     """Produce a conformance report you can hand to someone else.
 
@@ -162,7 +185,13 @@ def report(
         sys.exit(2)
 
     results_by_level = {
-        level: run(data, fmt, level, max_age_seconds=max_age)
+        level: run(
+            data,
+            fmt,
+            level,
+            max_age_seconds=max_age,
+            expected_nonce=expected_nonce,
+        )
         for level in range(max_level + 1)
     }
 
@@ -201,9 +230,7 @@ def report(
     if fail_under is not None:
         top = built.highest_level
         if top is None or top < fail_under:
-            click.echo(
-                f"Result: below the required Level {fail_under}", err=True
-            )
+            click.echo(f"Result: below the required Level {fail_under}", err=True)
             sys.exit(1)
 
 

src/trace_tests/modules/tr_rte.py

@@ -2,30 +2,35 @@
 
 from __future__ import annotations
 
+import hmac
 import re
 from typing import Any
 
 from trace_tests.result import Finding, Status
 
 _DIGEST_RE = re.compile(r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$")
-_VALID_PLATFORMS = frozenset({
-    "intel-tdx",
-    "amd-sev-snp",
-    # Azure confidential VM: SEV-SNP behind a Hyper-V paravisor (vTPM-rooted).
-    "azure-cvm-sev-snp",
-    "nvidia-h100",
-    "nvidia-blackwell",
-    "aws-nitro",
-    "arm-cca",
-    "google-confidential-space",
-    "tpm2",
-    "software-only",
-})
+_VALID_PLATFORMS = frozenset(
+    {
+        "intel-tdx",
+        "amd-sev-snp",
+        # Azure confidential VM: SEV-SNP behind a Hyper-V paravisor (vTPM-rooted).
+        "azure-cvm-sev-snp",
+        "nvidia-h100",
+        "nvidia-blackwell",
+        "aws-nitro",
+        "arm-cca",
+        "google-confidential-space",
+        "tpm2",
+        "software-only",
+    }
+)
 # Platforms that provide no hardware attestation evidence. Valid only at Level 0.
 _DEV_PLATFORMS = frozenset({"software-only"})
 
 
-def check(trace: dict[str, Any], level: int = 0) -> list[Finding]:
+def check(
+    trace: dict[str, Any], level: int = 0, expected_nonce: str | None = None
+) -> list[Finding]:
     """Return TR-RTE findings for the runtime / TEE platform claim.
 
     *level* is the conformance level being checked. Development-mode platforms
@@ -36,41 +41,105 @@ def check(trace: dict[str, Any], level: int = 0) -> list[Finding]:
     runtime = trace.get("runtime")
 
     if not isinstance(runtime, dict):
-        return [Finding("TR-RTE-001", Status.FAIL, "TR-RTE-001: runtime field is missing or not an object")]
+        return [
+            Finding(
+                "TR-RTE-001", Status.FAIL, "TR-RTE-001: runtime field is missing or not an object"
+            )
+        ]
 
     platform = runtime.get("platform")
     if platform in _DEV_PLATFORMS:
         if level == 0:
-            findings.append(Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})"))
+            findings.append(
+                Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})")
+            )
         else:
-            findings.append(Finding(
-                "TR-RTE-001", Status.FAIL,
-                f"TR-RTE-001: runtime.platform {platform!r} is development-mode and not acceptable for "
-                f"hardware-attested levels (Level {level} requires a hardware TEE platform)",
-            ))
+            findings.append(
+                Finding(
+                    "TR-RTE-001",
+                    Status.FAIL,
+                    f"TR-RTE-001: runtime.platform {platform!r} is development-mode "
+                    "and not acceptable for "
+                    f"hardware-attested levels (Level {level} requires a hardware TEE platform)",
+                )
+            )
     elif platform in _VALID_PLATFORMS:
-        findings.append(Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})"))
+        findings.append(
+            Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})")
+        )
     else:
-        findings.append(Finding(
-            "TR-RTE-001", Status.FAIL,
-            f"TR-RTE-001: runtime.platform {platform!r} is not in the registered set; valid: {sorted(_VALID_PLATFORMS)}",
-        ))
+        valid_platforms = sorted(_VALID_PLATFORMS)
+        findings.append(
+            Finding(
+                "TR-RTE-001",
+                Status.FAIL,
+                f"TR-RTE-001: runtime.platform {platform!r} is not in the "
+                f"registered set; valid: {valid_platforms}",
+            )
+        )
 
     measurement = runtime.get("measurement", "")
     if _DIGEST_RE.match(str(measurement)):
-        findings.append(Finding("TR-RTE-002", Status.PASS, "runtime.measurement has valid digest format"))
+        findings.append(
+            Finding("TR-RTE-002", Status.PASS, "runtime.measurement has valid digest format")
+        )
     else:
-        findings.append(Finding(
-            "TR-RTE-002", Status.FAIL,
-            f"TR-RTE-002: runtime.measurement must match sha256:<64hex> or sha384:<96hex>, got {measurement!r}",
-        ))
+        findings.append(
+            Finding(
+                "TR-RTE-002",
+                Status.FAIL,
+                "TR-RTE-002: runtime.measurement must match sha256:<64hex> or "
+                f"sha384:<96hex>, got {measurement!r}",
+            )
+        )
 
     rim_uri = runtime.get("rim_uri")
     if rim_uri is None:
-        findings.append(Finding("TR-RTE-003", Status.SKIP, "runtime.rim_uri not present (optional)"))
+        findings.append(
+            Finding("TR-RTE-003", Status.SKIP, "runtime.rim_uri not present (optional)")
+        )
     elif isinstance(rim_uri, str) and rim_uri.startswith("https://"):
-        findings.append(Finding("TR-RTE-003", Status.PASS, f"runtime.rim_uri is an https URI ({rim_uri[:60]})"))
+        findings.append(
+            Finding("TR-RTE-003", Status.PASS, f"runtime.rim_uri is an https URI ({rim_uri[:60]})")
+        )
     else:
-        findings.append(Finding("TR-RTE-003", Status.FAIL, f"TR-RTE-003: runtime.rim_uri must be an https URI, got {rim_uri!r}"))
+        findings.append(
+            Finding(
+                "TR-RTE-003",
+                Status.FAIL,
+                f"TR-RTE-003: runtime.rim_uri must be an https URI, got {rim_uri!r}",
+            )
+        )
+
+    if level >= 1:
+        actual_nonce = runtime.get("nonce")
+        if not isinstance(expected_nonce, str) or not expected_nonce:
+            findings.append(
+                Finding(
+                    "TR-RTE-004",
+                    Status.FAIL,
+                    "TR-RTE-004: Level 1+ verification requires the verifier's expected nonce",
+                )
+            )
+        elif not isinstance(actual_nonce, str) or not actual_nonce:
+            findings.append(
+                Finding(
+                    "TR-RTE-004",
+                    Status.FAIL,
+                    "TR-RTE-004: runtime.nonce is missing or empty",
+                )
+            )
+        elif hmac.compare_digest(actual_nonce, expected_nonce):
+            findings.append(
+                Finding("TR-RTE-004", Status.PASS, "runtime.nonce matches the verifier challenge")
+            )
+        else:
+            findings.append(
+                Finding(
+                    "TR-RTE-004",
+                    Status.FAIL,
+                    "TR-RTE-004: runtime.nonce does not match the verifier challenge",
+                )
+            )
 
     return findings

src/trace_tests/report.py

@@ -230,7 +230,7 @@ def badge_svg(data: ReportData) -> str:
     return (
         f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" '
         f'role="img" aria-label="{html.escape(left)}: {html.escape(right)}">'
-        f'<title>{html.escape(left)}: {html.escape(right)}</title>'
+        f"<title>{html.escape(left)}: {html.escape(right)}</title>"
         f'<rect width="{lw}" height="20" fill="#444"/>'
         f'<rect x="{lw}" width="{rw}" height="20" fill="{colour}"/>'
         f'<g fill="#fff" font-family="Verdana,DejaVu Sans,sans-serif" font-size="11">'
@@ -318,6 +318,7 @@ def to_html(data: ReportData) -> str:
     reproduce = (
         f"pip install agentrust-trace-tests=={e(data.suite_version)}\n"
         f"trace-tests verify --record &lt;your copy&gt; --level {top if top is not None else 0}"
+        + (" --expected-nonce &lt;verifier challenge&gt;" if top is not None and top >= 1 else "")
     )
 
     return f"""<!doctype html>
@@ -342,12 +343,12 @@ def to_html(data: ReportData) -> str:
 
 <h2>Levels</h2>
 <table><tr><th>Level</th><th>Name</th><th>Result</th><th>Detail</th></tr>
-{''.join(rows)}
+{"".join(rows)}
 </table>
 
 <h2>Findings</h2>
 <table><tr><th>Level</th><th>Module</th><th>Code</th><th>Status</th><th>Detail</th></tr>
-{''.join(finding_rows) or '<tr><td colspan="5">No findings.</td></tr>'}
+{"".join(finding_rows) or '<tr><td colspan="5">No findings.</td></tr>'}
 </table>
 
 <div class="note">

src/trace_tests/runner.py

@@ -21,6 +21,7 @@ def run(
     fmt: str,
     level: int,
     max_age_seconds: int = tr_env.DEFAULT_MAX_AGE_SECONDS,
+    expected_nonce: str | None = None,
 ) -> dict[str, list[Finding]]:
     """Run all modules required for *level* and return findings keyed by module ID."""
     if level not in _LEVEL_MODULES:
@@ -41,7 +42,7 @@ def run(
         results["TR-POL"] = tr_pol.check(trace)
 
     if "TR-RTE" in active:
-        results["TR-RTE"] = tr_rte.check(trace, level)
+        results["TR-RTE"] = tr_rte.check(trace, level, expected_nonce=expected_nonce)
 
     if "TR-SCA" in active:
         results["TR-SCA"] = tr_sca.check(trace)

tests/unit/test_cli.py

@@ -33,6 +33,12 @@ def test_unsigned_record_fails_level_1(fresh_level0_path):
     assert result.exit_code == 1, result.output
 
 
+def test_level_1_requires_verifier_nonce(fresh_level0_path):
+    result = CliRunner().invoke(main, ["verify", "--record", fresh_level0_path, "--level", "1"])
+    assert result.exit_code == 1, result.output
+    assert "requires the verifier's expected nonce" in result.output
+
+
 def test_unsigned_record_level_0_reports_unverified(fresh_level0_path):
     result = CliRunner().invoke(main, ["verify", "--record", fresh_level0_path, "--level", "0"])
     assert result.exit_code == 0, result.output