fix: make canonicalize() RFC 8785 conformant (#322)

docs/adr/0003-rfc9162-merkle-domain-separation.md

@@ -2,7 +2,7 @@
 
 **Status**: Accepted  
 **Date**: 2026-05-10  
-**Spec section**: Section 4.1.1, Section 3.2.2 (composite policy bundle), Section 3.2.3 (tool catalog hash), Section 3.2.5 (RAG corpus)
+**Spec section**: Section 4.1.1, Section 3.2.2 (composite policy bundle), Section 2.2.3 (composite policy bundle), Section 3.2.3 (tool catalog hash), Section 3.2.5 (RAG corpus)
 
 ## Context
 
@@ -15,10 +15,14 @@ Use the RFC 9162 (Certificate Transparency v2) Merkle tree construction with exp
 - Leaf nodes: `SHA-256(0x00 || leaf_data)`
 - Internal nodes: `SHA-256(0x01 || left_hash || right_hash)`
 
+This ADR defines three Merkle hash operations: tool catalog leaves, corpus document leaves, and composite policy sub-bundle leaves.
+
 Leaf data for tool entries: RFC 8785 canonical JSON of the tool descriptor (schema + description, sorted by tool name).  
 Leaf data for corpus documents: RFC 8785 canonical JSON of the document descriptor (hash + identifier + ingested_at).  
 Leaf data for composite policy sub-bundles (Section 3.2.2): the **raw digest bytes** of each sub-bundle hash, not the `sha256:`-prefixed hex string and not a JSON descriptor. Unlike the two above, this leaf carries no structured descriptor, because the ordering rule already fixes which sub-bundle each leaf is.
 
+Section 4.1.1 of the specification is the normative definition of the shared construction. Sections 3.2.2, 3.2.3, and 3.2.5.1 normatively define each artifact's leaf data and ordering; they take precedence over this ADR for those details. RFC 8785 applies only where those sections define JSON as an input to a hash. The composite policy sub-bundle leaf uses raw digest bytes and does not use JSON canonicalization.
+
 ## Rationale
 
 - RFC 9162 is a published IETF standard for Merkle tree construction, used in Certificate Transparency  -  a deployed, audited system
@@ -31,7 +35,7 @@ Leaf data for composite policy sub-bundles (Section 3.2.2): the **raw digest byt
 **Simple concatenation Merkle (no domain separation)**: Vulnerable to second-preimage attacks as described above. Rejected.
 
 **BLAKE3 Merkle**: BLAKE3 has built-in domain separation for its tree construction. Rejected because BLAKE3 is not yet in the standard library of all target languages, and SHA-256 is sufficient for this use case.
-
+id`. Composite policy sub-bundles sorted by policy language identifer (`cear`, `rego`, `yaml-agt), per Section 3.2.2
 **Flat hash (hash of concatenated hashes)**: Not a Merkle tree  -  does not support efficient membership proofs. Rejected because the spec's design supports future membership proof extensions.
 
 ## Consequences

python/tests/interop/test_trace_canonicalization_boundary.py

@@ -0,0 +1,60 @@
+"""Cross-repository RFC 8785 conformance guard (issue #322).
+
+trace-spec's canonicalization-boundary vectors are signed Trust Records whose
+signature verifies only over the RFC 8785 canonical bytes of every field
+except ``signature``. They are a black-box check on
+``agent_manifest._canonicalize.canonicalize`` from an independent producer:
+a non-conformant canonicalizer computes different signing bytes here and the
+signature stops verifying, which is exactly the failure issue #322 reported.
+
+The vectors are not vendored in this repository yet -- see
+``tests/interop/vectors/canonicalization-boundary/README.md`` for exact fetch
+commands. This test skips cleanly with those instructions until the files
+are present, and is not required for the rest of the suite to pass.
+"""
+from __future__ import annotations
+
+import base64
+import json
+from pathlib import Path
+
+import pytest
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
+
+from agent_manifest._canonicalize import canonicalize
+
+_VECTORS_DIR = Path(__file__).parent / "vectors" / "canonicalization-boundary"
+_README = _VECTORS_DIR / "README.md"
+
+
+def _b64url_decode(value: str) -> bytes:
+    return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+
+
+def _vector_files() -> list[Path]:
+    return sorted(_VECTORS_DIR.glob("*.json"))
+
+
+def test_canonicalize_matches_trace_spec_signature():
+    files = _vector_files()
+    if not files:
+        pytest.skip(f"vectors not vendored yet; see {_README}")
+
+    for path in files:
+        vector = json.loads(path.read_text(encoding="utf-8"))
+        record = vector["record"]
+        jwk = vector["trusted_key"]
+        body = {k: v for k, v in record.items() if k != "signature"}
+
+        pre_image = canonicalize(body)
+        public_key = Ed25519PublicKey.from_public_bytes(_b64url_decode(jwk["x"]))
+        signature = _b64url_decode(record["signature"])
+
+        try:
+            public_key.verify(signature, pre_image)
+        except InvalidSignature:
+            pytest.fail(
+                f"{path.name}: canonicalize() pre-image does not verify "
+                "against trace-spec's own signature over this record"
+            )

python/tests/interop/vectors/canonicalization-boundary/01-non-ascii-values.json

@@ -0,0 +1,54 @@
+{
+  "name": "non-ascii-values",
+  "description": "String values outside ASCII, all in the Basic Multilingual Plane. RFC 8785 emits them as literal UTF-8; a serializer that escapes to \\uXXXX signs different bytes and rejects this valid record.",
+  "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library",
+  "profile": "trace.canonicalization.boundary.v0",
+  "trusted_key": {
+    "kty": "OKP",
+    "crv": "Ed25519",
+    "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA"
+  },
+  "record": {
+    "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2",
+    "iat": 1785000000,
+    "subject": "spiffe://factory.example/agent/payments/prod",
+    "model": {
+      "provider": "anthropic",
+      "model_id": "claude-sonnet-4-6",
+      "version": "modèle-géant-4.6"
+    },
+    "runtime": {
+      "platform": "software-only",
+      "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+    },
+    "policy": {
+      "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "enforcement_mode": "enforce"
+    },
+    "data_class": "机密",
+    "build_provenance": {
+      "slsa_level": 0,
+      "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+    },
+    "appraisal": {
+      "status": "affirming",
+      "verifier": "https://verifier.example/v1"
+    },
+    "transparency": "https://rekor.example/api/v1/log/entries/0",
+    "cnf": {
+      "jwk": {
+        "kty": "OKP",
+        "crv": "Ed25519",
+        "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA"
+      }
+    },
+    "signature": "WehNEF0FqgUa_c85Hw7jbbz4_d_kg2GEyo4r4p242CNGjTkmmRNvVPuwTfjtKJwbOCuNspqEyrMNgZMOTh-OAA"
+  },
+  "expected": {
+    "outcome": "verified"
+  },
+  "diverges_under": [
+    "sort_keys_default",
+    "sort_keys_compact"
+  ]
+}

python/tests/interop/vectors/canonicalization-boundary/02-non-bmp-values.json

@@ -0,0 +1,54 @@
+{
+  "name": "non-bmp-values",
+  "description": "String values above U+FFFF, encoded as four UTF-8 bytes each. Under ASCII-escaping they become surrogate pairs; either way the bytes differ from RFC 8785's literal UTF-8.",
+  "spec": "trace-v0.2 section 3.2.2 — implementations MUST use an RFC 8785-conformant library",
+  "profile": "trace.canonicalization.boundary.v0",
+  "trusted_key": {
+    "kty": "OKP",
+    "crv": "Ed25519",
+    "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA"
+  },
+  "record": {
+    "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2",
+    "iat": 1785000000,
+    "subject": "spiffe://factory.example/agent/payments/prod",
+    "model": {
+      "provider": "anthropic",
+      "model_id": "claude-sonnet-4-6",
+      "version": "4.6-🤖"
+    },
+    "runtime": {
+      "platform": "software-only",
+      "measurement": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+    },
+    "policy": {
+      "bundle_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "enforcement_mode": "enforce"
+    },
+    "data_class": "confidential-🔒",
+    "build_provenance": {
+      "slsa_level": 0,
+      "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+    },
+    "appraisal": {
+      "status": "affirming",
+      "verifier": "https://verifier.example/v1"
+    },
+    "transparency": "https://rekor.example/api/v1/log/entries/0",
+    "cnf": {
+      "jwk": {
+        "kty": "OKP",
+        "crv": "Ed25519",
+        "x": "Be97jkxfFpVXzj9B-gwpMzv5t8PH30Edd-J7AIlrdoA"
+      }
+    },
+    "signature": "62CaOUWmDFPmgthTUkJ4cdwxmDQXzYg9hN6KaCB3EHjeDzeLiB_rdVFIRQrDVTzt-clmIoxNs7UxzJMFvWB_Bw"
+  },
+  "expected": {
+    "outcome": "verified"
+  },
+  "diverges_under": [
+    "sort_keys_default",
+    "sort_keys_compact"
+  ]
+}