From 9d83bdd4a017356881ea82924a4c5ff69e11ad16 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Sat, 22 Aug 2026 21:25:04 -0700 Subject: [PATCH 1/3] security: require verifier nonce for attested levels --- CHANGELOG.md | 9 ++ README.md | 3 +- docs/quickstart.md | 4 +- pyproject.toml | 2 +- src/trace_tests/cli.py | 41 +++++++-- src/trace_tests/modules/tr_rte.py | 137 ++++++++++++++++++++++-------- src/trace_tests/report.py | 7 +- src/trace_tests/runner.py | 3 +- tests/unit/test_cli.py | 6 ++ tests/unit/test_tr_rte.py | 27 +++++- 10 files changed, 187 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41aad00..bea2b6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +## v0.5.1 — 2026-08-22 + +- Level 1 and Level 2 verification now requires a verifier-issued challenge via + `--expected-nonce` and checks it against the signed `runtime.nonce` using + constant-time comparison. Previously nonce binding existed only as an + assertion over the repository's own pytest fixture; the shipped runner and + CLI could report conformance for a fresh signed record containing an + attacker-chosen or replayed nonce. + ## v0.5.0 — 2026-08-09 ### Added diff --git a/README.md b/README.md index 0c6d899..bd7b6b1 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ Seven test modules covering the full specification: envelope structure, signatur ```bash pip install agentrust-trace-tests -trace-tests verify --record path/to/trust-record.jwt --level 1 +trace-tests verify --record path/to/trust-record.jwt --level 1 \ + --expected-nonce "$VERIFIER_CHALLENGE" ``` ## A report you can hand to someone else diff --git a/docs/quickstart.md b/docs/quickstart.md index 3c7ebcd..a86e0cd 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -81,8 +81,8 @@ Level 0 is software-only (development). Level 1 requires TEE attestation. Level ```bash trace-tests verify --record sample-record.json --level 0 -trace-tests verify --record sample-record.json --level 1 -trace-tests verify --record sample-record.json --level 2 +trace-tests verify --record sample-record.json --level 1 --expected-nonce "$VERIFIER_CHALLENGE" +trace-tests verify --record sample-record.json --level 2 --expected-nonce "$VERIFIER_CHALLENGE" ``` The sample fixture passes Level 0. Levels 1 and 2 will fail on runtime attestation and transparency fields — that is expected. See [Trust Levels](levels.md) for what each level requires. diff --git a/pyproject.toml b/pyproject.toml index da3f9ad..3d50a3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agentrust-trace-tests" -version = "0.5.0" +version = "0.5.1" description = "TRACE conformance test suite" readme = "README.md" license = { text = "Apache-2.0" } diff --git a/src/trace_tests/cli.py b/src/trace_tests/cli.py index b01ed61..40dc8d1 100644 --- a/src/trace_tests/cli.py +++ b/src/trace_tests/cli.py @@ -89,7 +89,13 @@ def main() -> None: @main.command() @click.option("--record", required=True, type=click.Path(), help="Path to the trust record (JSON)") -@click.option("--level", default=0, type=click.IntRange(0, 2), show_default=True, help="Conformance level to check (0, 1, or 2)") +@click.option( + "--level", + default=0, + type=click.IntRange(0, 2), + show_default=True, + help="Conformance level to check (0, 1, or 2)", +) @click.option( "--max-age", "max_age", @@ -98,7 +104,12 @@ def main() -> None: show_default=True, help="Maximum allowed record age in seconds (iat freshness window)", ) -def verify(record: str, level: int, max_age: int) -> None: +@click.option( + "--expected-nonce", + default=None, + help="Verifier-issued challenge nonce; required for Level 1 and Level 2.", +) +def verify(record: str, level: int, max_age: int, expected_nonce: str | None) -> None: """Verify a TRACE trust record against the conformance suite.""" try: data, fmt = load_record(record) @@ -106,7 +117,13 @@ def verify(record: str, level: int, max_age: int) -> None: click.echo(f"Error: {exc}", err=True) sys.exit(2) - results = run(data, fmt, level, max_age_seconds=max_age) + results = run( + data, + fmt, + level, + max_age_seconds=max_age, + expected_nonce=expected_nonce, + ) exit_code = _print_report(record, fmt, level, results) sys.exit(exit_code) @@ -140,6 +157,11 @@ def verify(record: str, level: int, max_age: int) -> None: help="Exit non-zero unless the record reaches this level. Omit to always exit 0, " "which is what you want when generating an artifact rather than gating on one.", ) +@click.option( + "--expected-nonce", + default=None, + help="Verifier-issued challenge nonce; required for Level 1 and Level 2.", +) def report( record: str, max_level: int, @@ -148,6 +170,7 @@ def report( json_out: str | None, badge_out: str | None, fail_under: int | None, + expected_nonce: str | None, ) -> None: """Produce a conformance report you can hand to someone else. @@ -162,7 +185,13 @@ def report( sys.exit(2) results_by_level = { - level: run(data, fmt, level, max_age_seconds=max_age) + level: run( + data, + fmt, + level, + max_age_seconds=max_age, + expected_nonce=expected_nonce, + ) for level in range(max_level + 1) } @@ -201,9 +230,7 @@ def report( if fail_under is not None: top = built.highest_level if top is None or top < fail_under: - click.echo( - f"Result: below the required Level {fail_under}", err=True - ) + click.echo(f"Result: below the required Level {fail_under}", err=True) sys.exit(1) diff --git a/src/trace_tests/modules/tr_rte.py b/src/trace_tests/modules/tr_rte.py index 6184235..97aa93f 100644 --- a/src/trace_tests/modules/tr_rte.py +++ b/src/trace_tests/modules/tr_rte.py @@ -2,30 +2,35 @@ from __future__ import annotations +import hmac import re from typing import Any from trace_tests.result import Finding, Status _DIGEST_RE = re.compile(r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$") -_VALID_PLATFORMS = frozenset({ - "intel-tdx", - "amd-sev-snp", - # Azure confidential VM: SEV-SNP behind a Hyper-V paravisor (vTPM-rooted). - "azure-cvm-sev-snp", - "nvidia-h100", - "nvidia-blackwell", - "aws-nitro", - "arm-cca", - "google-confidential-space", - "tpm2", - "software-only", -}) +_VALID_PLATFORMS = frozenset( + { + "intel-tdx", + "amd-sev-snp", + # Azure confidential VM: SEV-SNP behind a Hyper-V paravisor (vTPM-rooted). + "azure-cvm-sev-snp", + "nvidia-h100", + "nvidia-blackwell", + "aws-nitro", + "arm-cca", + "google-confidential-space", + "tpm2", + "software-only", + } +) # Platforms that provide no hardware attestation evidence. Valid only at Level 0. _DEV_PLATFORMS = frozenset({"software-only"}) -def check(trace: dict[str, Any], level: int = 0) -> list[Finding]: +def check( + trace: dict[str, Any], level: int = 0, expected_nonce: str | None = None +) -> list[Finding]: """Return TR-RTE findings for the runtime / TEE platform claim. *level* is the conformance level being checked. Development-mode platforms @@ -36,41 +41,105 @@ def check(trace: dict[str, Any], level: int = 0) -> list[Finding]: runtime = trace.get("runtime") if not isinstance(runtime, dict): - return [Finding("TR-RTE-001", Status.FAIL, "TR-RTE-001: runtime field is missing or not an object")] + return [ + Finding( + "TR-RTE-001", Status.FAIL, "TR-RTE-001: runtime field is missing or not an object" + ) + ] platform = runtime.get("platform") if platform in _DEV_PLATFORMS: if level == 0: - findings.append(Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})")) + findings.append( + Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})") + ) else: - findings.append(Finding( - "TR-RTE-001", Status.FAIL, - f"TR-RTE-001: runtime.platform {platform!r} is development-mode and not acceptable for " - f"hardware-attested levels (Level {level} requires a hardware TEE platform)", - )) + findings.append( + Finding( + "TR-RTE-001", + Status.FAIL, + f"TR-RTE-001: runtime.platform {platform!r} is development-mode " + "and not acceptable for " + f"hardware-attested levels (Level {level} requires a hardware TEE platform)", + ) + ) elif platform in _VALID_PLATFORMS: - findings.append(Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})")) + findings.append( + Finding("TR-RTE-001", Status.PASS, f"runtime.platform is registered ({platform!r})") + ) else: - findings.append(Finding( - "TR-RTE-001", Status.FAIL, - f"TR-RTE-001: runtime.platform {platform!r} is not in the registered set; valid: {sorted(_VALID_PLATFORMS)}", - )) + valid_platforms = sorted(_VALID_PLATFORMS) + findings.append( + Finding( + "TR-RTE-001", + Status.FAIL, + f"TR-RTE-001: runtime.platform {platform!r} is not in the " + f"registered set; valid: {valid_platforms}", + ) + ) measurement = runtime.get("measurement", "") if _DIGEST_RE.match(str(measurement)): - findings.append(Finding("TR-RTE-002", Status.PASS, "runtime.measurement has valid digest format")) + findings.append( + Finding("TR-RTE-002", Status.PASS, "runtime.measurement has valid digest format") + ) else: - findings.append(Finding( - "TR-RTE-002", Status.FAIL, - f"TR-RTE-002: runtime.measurement must match sha256:<64hex> or sha384:<96hex>, got {measurement!r}", - )) + findings.append( + Finding( + "TR-RTE-002", + Status.FAIL, + "TR-RTE-002: runtime.measurement must match sha256:<64hex> or " + f"sha384:<96hex>, got {measurement!r}", + ) + ) rim_uri = runtime.get("rim_uri") if rim_uri is None: - findings.append(Finding("TR-RTE-003", Status.SKIP, "runtime.rim_uri not present (optional)")) + findings.append( + Finding("TR-RTE-003", Status.SKIP, "runtime.rim_uri not present (optional)") + ) elif isinstance(rim_uri, str) and rim_uri.startswith("https://"): - findings.append(Finding("TR-RTE-003", Status.PASS, f"runtime.rim_uri is an https URI ({rim_uri[:60]})")) + findings.append( + Finding("TR-RTE-003", Status.PASS, f"runtime.rim_uri is an https URI ({rim_uri[:60]})") + ) else: - findings.append(Finding("TR-RTE-003", Status.FAIL, f"TR-RTE-003: runtime.rim_uri must be an https URI, got {rim_uri!r}")) + findings.append( + Finding( + "TR-RTE-003", + Status.FAIL, + f"TR-RTE-003: runtime.rim_uri must be an https URI, got {rim_uri!r}", + ) + ) + + if level >= 1: + actual_nonce = runtime.get("nonce") + if not isinstance(expected_nonce, str) or not expected_nonce: + findings.append( + Finding( + "TR-RTE-004", + Status.FAIL, + "TR-RTE-004: Level 1+ verification requires the verifier's expected nonce", + ) + ) + elif not isinstance(actual_nonce, str) or not actual_nonce: + findings.append( + Finding( + "TR-RTE-004", + Status.FAIL, + "TR-RTE-004: runtime.nonce is missing or empty", + ) + ) + elif hmac.compare_digest(actual_nonce, expected_nonce): + findings.append( + Finding("TR-RTE-004", Status.PASS, "runtime.nonce matches the verifier challenge") + ) + else: + findings.append( + Finding( + "TR-RTE-004", + Status.FAIL, + "TR-RTE-004: runtime.nonce does not match the verifier challenge", + ) + ) return findings diff --git a/src/trace_tests/report.py b/src/trace_tests/report.py index a55a607..686235c 100644 --- a/src/trace_tests/report.py +++ b/src/trace_tests/report.py @@ -230,7 +230,7 @@ def badge_svg(data: ReportData) -> str: return ( f'' - f'{html.escape(left)}: {html.escape(right)}' + f"{html.escape(left)}: {html.escape(right)}" f'' f'' f'' @@ -318,6 +318,7 @@ def to_html(data: ReportData) -> str: reproduce = ( f"pip install agentrust-trace-tests=={e(data.suite_version)}\n" f"trace-tests verify --record <your copy> --level {top if top is not None else 0}" + + (" --expected-nonce <verifier challenge>" if top is not None and top >= 1 else "") ) return f""" @@ -342,12 +343,12 @@ def to_html(data: ReportData) -> str:

Levels

-{''.join(rows)} +{"".join(rows)}
LevelNameResultDetail

Findings

-{''.join(finding_rows) or ''} +{"".join(finding_rows) or ''}
LevelModuleCodeStatusDetail
No findings.
No findings.
diff --git a/src/trace_tests/runner.py b/src/trace_tests/runner.py index 036dc12..8689ef0 100644 --- a/src/trace_tests/runner.py +++ b/src/trace_tests/runner.py @@ -21,6 +21,7 @@ def run( fmt: str, level: int, max_age_seconds: int = tr_env.DEFAULT_MAX_AGE_SECONDS, + expected_nonce: str | None = None, ) -> dict[str, list[Finding]]: """Run all modules required for *level* and return findings keyed by module ID.""" if level not in _LEVEL_MODULES: @@ -41,7 +42,7 @@ def run( results["TR-POL"] = tr_pol.check(trace) if "TR-RTE" in active: - results["TR-RTE"] = tr_rte.check(trace, level) + results["TR-RTE"] = tr_rte.check(trace, level, expected_nonce=expected_nonce) if "TR-SCA" in active: results["TR-SCA"] = tr_sca.check(trace) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index e9f9f9d..3c20483 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -33,6 +33,12 @@ def test_unsigned_record_fails_level_1(fresh_level0_path): assert result.exit_code == 1, result.output +def test_level_1_requires_verifier_nonce(fresh_level0_path): + result = CliRunner().invoke(main, ["verify", "--record", fresh_level0_path, "--level", "1"]) + assert result.exit_code == 1, result.output + assert "requires the verifier's expected nonce" in result.output + + def test_unsigned_record_level_0_reports_unverified(fresh_level0_path): result = CliRunner().invoke(main, ["verify", "--record", fresh_level0_path, "--level", "0"]) assert result.exit_code == 0, result.output diff --git a/tests/unit/test_tr_rte.py b/tests/unit/test_tr_rte.py index 16754a1..6dfbf48 100644 --- a/tests/unit/test_tr_rte.py +++ b/tests/unit/test_tr_rte.py @@ -1,8 +1,8 @@ """Unit tests for TR-RTE module.""" import pytest + from trace_tests.modules.tr_rte import check -from trace_tests.result import Status _VALID = { "runtime": { @@ -19,8 +19,14 @@ def test_valid_runtime_passes(): def test_azure_cvm_platform_passes(): """azure-cvm-sev-snp is a recognized hardware platform (vTPM-rooted SEV-SNP).""" - trace = {"runtime": {"platform": "azure-cvm-sev-snp", "measurement": "sha384:" + "a" * 96}} - findings = check(trace, level=2) + trace = { + "runtime": { + "platform": "azure-cvm-sev-snp", + "measurement": "sha384:" + "a" * 96, + "nonce": "challenge", + } + } + findings = check(trace, level=2, expected_nonce="challenge") assert all(not f.failed() for f in findings), findings @@ -69,3 +75,18 @@ def test_http_rim_uri_fails(): trace = {"runtime": {**_VALID["runtime"], "rim_uri": "http://example.org/rim/tdx-v1"}} codes = {f.code for f in check(trace) if f.failed()} assert "TR-RTE-003" in codes, "plain http rim_uri must be rejected; https only" + + +@pytest.mark.parametrize("level", [1, 2]) +def test_attested_levels_require_expected_nonce(level): + trace = {"runtime": {**_VALID["runtime"], "nonce": "record-chosen"}} + codes = {f.code for f in check(trace, level=level) if f.failed()} + assert "TR-RTE-004" in codes + + +def test_nonce_mismatch_fails_and_match_passes(): + trace = {"runtime": {**_VALID["runtime"], "nonce": "signed-nonce"}} + mismatch = check(trace, level=1, expected_nonce="verifier-challenge") + assert any(f.code == "TR-RTE-004" and f.failed() for f in mismatch) + match = check(trace, level=1, expected_nonce="signed-nonce") + assert any(f.code == "TR-RTE-004" and f.passed() for f in match) From 719323c402b77488971b2a726b427aa7430edb7d Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Sun, 23 Aug 2026 10:52:56 -0700 Subject: [PATCH 2/3] ci: fail closed when CodeQL analysis fails --- .github/workflows/codeql.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8743fe3..ecb8832 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,6 +38,5 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4.37.7 - continue-on-error: true with: category: /language:python From 32a7769495cfb693498be698324fb1bfd3ce25c5 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Sun, 23 Aug 2026 11:45:52 -0700 Subject: [PATCH 3/3] docs: document verifier nonce finding --- docs/error-codes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/error-codes.md b/docs/error-codes.md index 032bace..66f3edb 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -27,6 +27,7 @@ All TRACE test failures emit a structured error code of the form `TR--