diff --git a/docs/adr/0003-rfc9162-merkle-domain-separation.md b/docs/adr/0003-rfc9162-merkle-domain-separation.md index bc39f441..2e710bfd 100644 --- a/docs/adr/0003-rfc9162-merkle-domain-separation.md +++ b/docs/adr/0003-rfc9162-merkle-domain-separation.md @@ -2,7 +2,7 @@ **Status**: Accepted **Date**: 2026-05-10 -**Spec section**: Section 4.1.1, Section 3.2.2 (composite policy bundle), Section 3.2.3 (tool catalog hash), Section 3.2.5 (RAG corpus) +**Spec section**: Section 4.1.1, Section 3.2.2 (composite policy bundle), Section 2.2.3 (composite policy bundle), Section 3.2.3 (tool catalog hash), Section 3.2.5 (RAG corpus) ## Context @@ -19,6 +19,14 @@ Leaf data for tool entries: RFC 8785 canonical JSON of the tool descriptor (sche Leaf data for corpus documents: RFC 8785 canonical JSON of the document descriptor (hash + identifier + ingested_at). Leaf data for composite policy sub-bundles (Section 3.2.2): the **raw digest bytes** of each sub-bundle hash, not the `sha256:`-prefixed hex string and not a JSON descriptor. Unlike the two above, this leaf carries no structured descriptor, because the ordering rule already fixes which sub-bundle each leaf is. +This ADR defines three Merkle hash operations: tool catalog leaves, corpus document leaves, and composite policy sub-bundle leaves. + +Leaf data for tool entries: RFC 8785 canonical JSON of the tool descriptor (schema + description, sorted by tool name). +Leaf data for corpus documents: RFC 8785 canonical JSON of the document descriptor (hash + identifier + ingested_at). +Leaf data for composite policy sub-bundles (Section 3.2.2): the **raw digest bytes** of each sub-bundle hash, not the `sha256:`-prefixed hex string and not a JSON descriptor. Unlike the two above, this leaf carries no structured descriptor, because the ordering rule already fixes which sub-bundle each leaf is. + +Section 4.1.1 of the specification is the normative definition of the shared construction. Sections 3.2.2, 3.2.3, and 3.2.5.1 normatively define each artifact's leaf data and ordering; they take precedence over this ADR for those details. RFC 8785 applies only where those sections define JSON as an input to a hash. The composite policy sub-bundle leaf uses raw digest bytes and does not use JSON canonicalization. + ## Rationale - RFC 9162 is a published IETF standard for Merkle tree construction, used in Certificate Transparency - a deployed, audited system @@ -31,7 +39,7 @@ Leaf data for composite policy sub-bundles (Section 3.2.2): the **raw digest byt **Simple concatenation Merkle (no domain separation)**: Vulnerable to second-preimage attacks as described above. Rejected. **BLAKE3 Merkle**: BLAKE3 has built-in domain separation for its tree construction. Rejected because BLAKE3 is not yet in the standard library of all target languages, and SHA-256 is sufficient for this use case. - +id`. Composite policy sub-bundles sorted by policy language identifer (`cear`, `rego`, `yaml-agt), per Section 3.2.2 **Flat hash (hash of concatenated hashes)**: Not a Merkle tree - does not support efficient membership proofs. Rejected because the spec's design supports future membership proof extensions. ## Consequences diff --git a/python/tests/interop/test_trace_canonicalization_boundary.py b/python/tests/interop/test_trace_canonicalization_boundary.py new file mode 100644 index 00000000..da8723f8 --- /dev/null +++ b/python/tests/interop/test_trace_canonicalization_boundary.py @@ -0,0 +1,60 @@ +"""Cross-repository RFC 8785 conformance guard (issue #322). + +trace-spec's canonicalization-boundary vectors are signed Trust Records whose +signature verifies only over the RFC 8785 canonical bytes of every field +except ``signature``. They are a black-box check on +``agent_manifest._canonicalize.canonicalize`` from an independent producer: +a non-conformant canonicalizer computes different signing bytes here and the +signature stops verifying, which is exactly the failure issue #322 reported. + +The vectors are not vendored in this repository yet -- see +``tests/interop/vectors/canonicalization-boundary/README.md`` for exact fetch +commands. This test skips cleanly with those instructions until the files +are present, and is not required for the rest of the suite to pass. +""" +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import pytest +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from agent_manifest._canonicalize import canonicalize + +_VECTORS_DIR = Path(__file__).parent / "vectors" / "canonicalization-boundary" +_README = _VECTORS_DIR / "README.md" + + +def _b64url_decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _vector_files() -> list[Path]: + return sorted(_VECTORS_DIR.glob("*.json")) + + +def test_canonicalize_matches_trace_spec_signature(): + files = _vector_files() + if not files: + pytest.skip(f"vectors not vendored yet; see {_README}") + + for path in files: + vector = json.loads(path.read_text(encoding="utf-8")) + record = vector["record"] + jwk = vector["trusted_key"] + body = {k: v for k, v in record.items() if k != "signature"} + + pre_image = canonicalize(body) + public_key = Ed25519PublicKey.from_public_bytes(_b64url_decode(jwk["x"])) + signature = _b64url_decode(record["signature"]) + + try: + public_key.verify(signature, pre_image) + except InvalidSignature: + pytest.fail( + f"{path.name}: canonicalize() pre-image does not verify " + "against trace-spec's own signature over this record" + ) diff --git a/python/tests/interop/vectors/canonicalization-boundary/01-non-ascii-values.json b/python/tests/interop/vectors/canonicalization-boundary/01-non-ascii-values.json new file mode 100644 index 00000000..83d0c805 --- /dev/null +++ b/python/tests/interop/vectors/canonicalization-boundary/01-non-ascii-values.json @@ -0,0 +1,54 @@ +{ + "name": "non-ascii-values", + "description": "String values outside ASCII, all in the Basic Multilingual Plane. RFC 8785 emits them as literal UTF-8; a serializer that escapes to \\uXXXX signs different bytes and rejects this valid record.", + "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library", + "profile": "trace.canonicalization.boundary.v0", + "trusted_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + }, + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1785000000, + "subject": "spiffe://factory.example/agent/payments/prod", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-6", + "version": "modèle-géant-4.6" + }, + "runtime": { + "platform": "software-only", + "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, + "policy": { + "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "enforcement_mode": "enforce" + }, + "data_class": "机密", + "build_provenance": { + "slsa_level": 0, + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example/v1" + }, + "transparency": "https://rekor.example/api/v1/log/entries/0", + "cnf": { + "jwk": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + } + }, + "signature": "WehNEF0FqgUa_c85Hw7jbbz4_d_kg2GEyo4r4p242CNGjTkmmRNvVPuwTfjtKJwbOCuNspqEyrMNgZMOTh-OAA" + }, + "expected": { + "outcome": "verified" + }, + "diverges_under": [ + "sort_keys_default", + "sort_keys_compact" + ] +} \ No newline at end of file diff --git a/python/tests/interop/vectors/canonicalization-boundary/02-non-bmp-values.json b/python/tests/interop/vectors/canonicalization-boundary/02-non-bmp-values.json new file mode 100644 index 00000000..5778184b --- /dev/null +++ b/python/tests/interop/vectors/canonicalization-boundary/02-non-bmp-values.json @@ -0,0 +1,54 @@ +{ + "name": "non-bmp-values", + "description": "String values above U+FFFF, encoded as four UTF-8 bytes each. Under ASCII-escaping they become surrogate pairs; either way the bytes differ from RFC 8785's literal UTF-8.", + "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library", + "profile": "trace.canonicalization.boundary.v0", + "trusted_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + }, + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1785000000, + "subject": "spiffe://factory.example/agent/payments/prod", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-6", + "version": "4.6-🤖" + }, + "runtime": { + "platform": "software-only", + "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, + "policy": { + "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "enforcement_mode": "enforce" + }, + "data_class": "confidential-🔒", + "build_provenance": { + "slsa_level": 0, + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example/v1" + }, + "transparency": "https://rekor.example/api/v1/log/entries/0", + "cnf": { + "jwk": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + } + }, + "signature": "62CaOUWmDFPmgthTUkJ4cdwxmDQXzYg9hN6KaCB3EHjeDzeLiB_rdVFIRQrDVTzt-clmIoxNs7UxzJMFvWB_Bw" + }, + "expected": { + "outcome": "verified" + }, + "diverges_under": [ + "sort_keys_default", + "sort_keys_compact" + ] +} \ No newline at end of file diff --git a/python/tests/interop/vectors/canonicalization-boundary/03-utf16-key-order.json b/python/tests/interop/vectors/canonicalization-boundary/03-utf16-key-order.json new file mode 100644 index 00000000..ea8d80b0 --- /dev/null +++ b/python/tests/interop/vectors/canonicalization-boundary/03-utf16-key-order.json @@ -0,0 +1,56 @@ +{ + "name": "utf16-key-order", + "description": "Two object keys whose order under RFC 8785's UTF-16 code-unit sort is the reverse of their code-point order. This is the record that distinguishes a true RFC 8785 serializer from json.dumps with every option set carefully: compact separators and ensure_ascii=False survive vectors 01 and 02, and fail here.", + "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library", + "profile": "trace.canonicalization.boundary.v0", + "trusted_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + }, + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1785000000, + "subject": "spiffe://factory.example/agent/payments/prod", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-6" + }, + "runtime": { + "platform": "software-only", + "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, + "policy": { + "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 0, + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example/v1" + }, + "transparency": "https://rekor.example/api/v1/log/entries/0", + "cnf": { + "jwk": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA", + "zk😀": "sorts-first-under-rfc-8785", + "zk�": "sorts-second-under-rfc-8785" + } + }, + "signature": "CjOuPwCnxnwegFjguiSCi-_xPg3iOwnCgyKuKYnV0OorofjPJrkOLn3dUFa-6tVf0z8EDiHaczl6AN46MuBtCQ" + }, + "expected": { + "outcome": "verified" + }, + "diverges_under": [ + "sort_keys_default", + "sort_keys_compact", + "sort_keys_compact_utf8" + ] +} \ No newline at end of file diff --git a/python/tests/interop/vectors/canonicalization-boundary/04-utf16-key-order-nested.json b/python/tests/interop/vectors/canonicalization-boundary/04-utf16-key-order-nested.json new file mode 100644 index 00000000..03512bc0 --- /dev/null +++ b/python/tests/interop/vectors/canonicalization-boundary/04-utf16-key-order-nested.json @@ -0,0 +1,58 @@ +{ + "name": "utf16-key-order-nested", + "description": "The divergence of vector 03 moved inside a nested object, so that a canonicalizer sorting by UTF-16 code units at the outer levels and by code points below them passes 03 and fails here. Without it the closest non-conformant form is caught by one vector, and the boundary disappears with that vector.", + "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library", + "profile": "trace.canonicalization.boundary.v0", + "trusted_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA" + }, + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1785000000, + "subject": "spiffe://factory.example/agent/payments/prod", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-6" + }, + "runtime": { + "platform": "software-only", + "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, + "policy": { + "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 0, + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example/v1" + }, + "transparency": "https://rekor.example/api/v1/log/entries/0", + "cnf": { + "jwk": { + "kty": "OKP", + "crv": "Ed25519", + "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA", + "zmeta": { + "zk😀": "sorts-first-under-rfc-8785", + "zk�": "sorts-second-under-rfc-8785" + } + } + }, + "signature": "yXsht9nU--Hvr8K7xHq72MOU6xyVhsCKw0_YcAdDff641JNlPG1d2qAZ_zwXaLe48agijvRk3MVZioG85aAiBg" + }, + "expected": { + "outcome": "verified" + }, + "diverges_under": [ + "sort_keys_default", + "sort_keys_compact", + "sort_keys_compact_utf8" + ] +} \ No newline at end of file diff --git a/python/tests/interop/vectors/canonicalization-boundary/README.md b/python/tests/interop/vectors/canonicalization-boundary/README.md new file mode 100644 index 00000000..2c7206dd --- /dev/null +++ b/python/tests/interop/vectors/canonicalization-boundary/README.md @@ -0,0 +1,26 @@ +# Vendored trace-spec canonicalization-boundary vectors + +Empty until fetched. `test_trace_canonicalization_boundary.py` skips with +fetch instructions when no `*.json` files are present here. + +Source: https://github.com/agentrust-io/trace-spec/tree/main/examples/canonicalization-boundary + +These are signed fixtures ΓÇö fetch the raw bytes directly rather than +retyping them, since the guard exists to catch canonicalization differences +that a retyped copy could silently hide. + +```powershell +git clone --depth 1 https://github.com/agentrust-io/trace-spec C:\Temp\trace-spec +Copy-Item C:\Temp\trace-spec\examples\canonicalization-boundary\*.json . +``` + +or per file: + +```powershell +curl.exe -o 01-non-ascii-values.json https://raw.githubusercontent.com/agentrust-io/trace-spec/main/examples/canonicalization-boundary/01-non-ascii-values.json +curl.exe -o 02-non-bmp-values.json https://raw.githubusercontent.com/agentrust-io/trace-spec/main/examples/canonicalization-boundary/02-non-bmp-values.json +curl.exe -o 03-utf16-key-order.json https://raw.githubusercontent.com/agentrust-io/trace-spec/main/examples/canonicalization-boundary/03-utf16-key-order.json +curl.exe -o 04-utf16-key-order-nested.json https://raw.githubusercontent.com/agentrust-io/trace-spec/main/examples/canonicalization-boundary/04-utf16-key-order-nested.json +``` + +Run `pytest tests/interop/test_trace_canonicalization_boundary.py -v` afterward.