feat(arx): base64url wire encoding (B. prefix) for chat-safe ARX

docs/architecture.md

@@ -88,7 +88,7 @@ The fragment protocol keeps the JSON envelope stable and treats compression stri
 - `plain` stores base64url-encoded JSON for compatibility and debugging
 - `lz` stores compressed JSON via `lz-string` when it produces a smaller fragment
 - `deflate` stores deflate-compressed UTF-8 JSON bytes when it outperforms other codecs
-- `arx` applies domain-dictionary substitution, brotli compression (quality 11), and binary-to-text encoding for best-in-class compression. Three encoding tiers are used: base76 (ASCII, 77 fragment-safe chars), base1k (Unicode, 1774 chars from U+00A1–U+07FF), and baseBMP (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). The encoder tries all three and picks the shortest — baseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76, achieving ~70% smaller fragments than deflate on typical payloads (6.13x compression ratio for 8k markdown). Full pipeline timing is ~10ms for 8k payloads. The substitution dictionary is served as a static file at `/arx-dictionary.json` so agents can fetch it for local compression; a pre-compressed `/arx-dictionary.json.br` variant is also available. The viewer loads the dictionary on startup and falls back to a built-in table if the fetch fails.
+- `arx` applies domain-dictionary substitution, brotli compression (quality 11), and binary-to-text encoding for best-in-class compression. Four wire shapes are candidates: base76 (ASCII, 77 fragment-safe chars), base64url (RFC 4648 `A-Za-z0-9-_` with a `B.` prefix for detection), base1k (Unicode, 1774 chars from U+00A1–U+07FF), and baseBMP (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). The async encoder tries all four and picks the shortest **transport** length (percent-encoded UTF-8 length for non-ASCII), so base64url can win over Unicode encodings on chat-style surfaces. baseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76 for the same compressed bytes, achieving ~70% smaller fragments than deflate on typical payloads (~6.1x compression ratio for 8k markdown). Full pipeline timing is on the order of ~8–14ms for 8k payloads depending on the wire encoding. The substitution dictionary is served as a static file at `/arx-dictionary.json` so agents can fetch it for local compression; a pre-compressed `/arx-dictionary.json.br` variant is also available. The viewer loads the dictionary on startup and falls back to a built-in table if the fetch fails.
 - packed wire mode (`p: 1`) shortens transport keys before compression, then unpacks back to the standard envelope during decode
 - automatic async codec selection tries `arx -> deflate -> lz -> plain` and compares packed + non-packed candidates
 - sync codec selection (used by examples and legacy paths) tries `deflate -> lz -> plain`

docs/payload-format.md

@@ -18,7 +18,7 @@ Supported codecs:
 - `plain` - base64url-encoded JSON
 - `lz` - `lz-string` compressed JSON encoded for URL-safe transport
 - `deflate` - deflate-compressed UTF-8 JSON bytes encoded as base64url
-- `arx` - domain-dictionary substitution + brotli (quality 11) + binary-to-text encoding. arx fragments include dictionary version metadata in the outer format (`v1.arx.<dictVersion>.<payload>`) so links stay portable across dictionary updates. Three encoding tiers are supported: **base76** (ASCII-only, 77 fragment-safe chars), **base1k** (Unicode, 1774 chars from U+00A1–U+07FF), and **baseBMP** (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). The encoder tries all three and picks the shortest. BaseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76 for the same compressed bytes. BaseBMP payloads are prefixed with a U+FFF0 marker for detection. The substitution dictionary is served at `/arx-dictionary.json` (with a pre-compressed `/arx-dictionary.json.br` variant) so agents can fetch it for local compression.
+- `arx` - domain-dictionary substitution + brotli (quality 11) + binary-to-text encoding. arx fragments include dictionary version metadata in the outer format (`v1.arx.<dictVersion>.<payload>`) so links stay portable across dictionary updates. Four wire shapes are tried and the shortest **transport** size wins (see `computeTransportLength` in `fragment.ts` — non-ASCII Unicode may count longer after percent-encoding): **base76** (ASCII-only, 77 fragment-safe chars), **base64url** (standard RFC 4648 alphabet `A-Za-z0-9-_`, no padding, prefixed with `B.` for detection), **base1k** (Unicode, 1774 chars from U+00A1–U+07FF), and **baseBMP** (high-density Unicode, ~62k safe BMP code points from U+00A1–U+FFEF, ~15.92 bits/char). BaseBMP produces ~32% fewer characters than base1k and ~55% fewer than base76 for the same compressed bytes. BaseBMP payloads are prefixed with a U+FFF0 marker for detection. The viewer’s `arxDecompress` auto-detects the wire shape (including the rare case where a base76 length prefix is also `B.` — it tries base64url first and falls back to base76 if Brotli fails). The substitution dictionary is served at `/arx-dictionary.json` (with a pre-compressed `/arx-dictionary.json.br` variant) so agents can fetch it for local compression.
 
 The encoder now also supports a packed wire representation (`p: 1`) that shortens key names before compression. Packed mode is transport-only; decoded envelopes normalize back to the standard shape.
 
@@ -95,17 +95,19 @@ Running `npm run codec:poc` (single markdown artifact containing `AGENTS.md`) cu
 - `lz+packed`: ~5,674 chars
 - `deflate`: ~4,392 chars
 - `deflate+packed`: ~4,375 chars
-- `arx` (base76): ~3,311 chars
-- `arx` (base1k): ~1,923 chars
-- `arx` (baseBMP): ~1,306 chars (best)
+- `arx` (base76): ~3,336 chars
+- `arx` (base64url): ~3,485 chars
+- `arx` (base1k): ~1,938 chars
+- `arx` (baseBMP): ~1,316 chars (best raw char count)
 
-Result: `arx` with baseBMP encoding achieves ~70% smaller fragments than `deflate` on this payload (6.13x compression ratio). The improvement comes from brotli compression (~20% better than deflate), baseBMP encoding (~15.92 bits/char using ~62k safe BMP code points), and domain dictionary substitution. Base1k and ASCII base76 remain available as fallbacks for environments that cannot handle wide Unicode in URL fragments.
+Result: `arx` with baseBMP encoding achieves ~69% smaller fragments than `deflate` on this payload (~6.1x compression ratio). The improvement comes from brotli compression (~20% better than deflate), baseBMP encoding (~15.92 bits/char using ~62k safe BMP code points), and domain dictionary substitution. **base64url** is an ASCII-only option that can beat base76 on surfaces that percent-encode Unicode (chat apps, some shorteners). Base1k, baseBMP, and base76 remain available; auto-selection compares estimated transport length.
 
 Timing (AGENTS.md 8000 chars, avg of 10 runs):
 - `deflate+base64url`: ~0.1ms
-- `arx+base76`: ~14.6ms
-- `arx+base1k`: ~11.1ms
-- `arx+baseBMP`: ~9.7ms
+- `arx+base76`: ~13.8ms
+- `arx+base64url`: ~8.1ms
+- `arx+base1k`: ~12.0ms
+- `arx+baseBMP`: ~10.8ms
 
 ## Active artifact behavior
 

scripts/codec-poc.mjs

@@ -195,6 +195,11 @@ function encBaseBMP(bytes) {
   return "\uFFF0" + BMP[Math.floor(bytes.length / BMP.length)] + BMP[bytes.length % BMP.length] + c.join("");
 }
 
+function encBase64url(bytes) {
+  if (!bytes.length) return "B.";
+  return `B.${Buffer.from(bytes).toString("base64url")}`;
+}
+
 // --- Dictionary substitution ---
 const SBC = [1,2,3,4,5,6,7,8,0x0b,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d];
 const subs = [];
@@ -248,12 +253,18 @@ for (const { name, text } of payloads) {
   const bbmp = encBaseBMP(compressed);
   const msBmp = performance.now() - t0_bmp;
 
+  const t0_b64 = performance.now();
+  const b64u = encBase64url(compressed);
+  const msB64 = performance.now() - t0_b64;
+
   console.log(`  Deflate+base64url:  ${deflateB64.length} chars  (${deflateMs.toFixed(1)}ms)`);
   console.log(`  ARX ASCII (base76): ${b76.length} chars  (brotli ${brotliMs.toFixed(1)}ms + encode ${ms76.toFixed(1)}ms)`);
+  console.log(`  ARX base64url:      ${b64u.length} chars  (brotli ${brotliMs.toFixed(1)}ms + encode ${msB64.toFixed(1)}ms)`);
   console.log(`  ARX Unicode (1k):   ${b1k.length} chars  (brotli ${brotliMs.toFixed(1)}ms + encode ${ms1k.toFixed(1)}ms)`);
   console.log(`  ARX Unicode (BMP):  ${bbmp.length} chars  (brotli ${brotliMs.toFixed(1)}ms + encode ${msBmp.toFixed(1)}ms)`);
   console.log('');
   console.log(`  vs deflate:  base76 ${((1 - b76.length/deflateB64.length)*100).toFixed(1)}% smaller`);
+  console.log(`               base64url ${((1 - b64u.length/deflateB64.length)*100).toFixed(1)}% smaller`);
   console.log(`               base1k ${((1 - b1k.length/deflateB64.length)*100).toFixed(1)}% smaller`);
   console.log(`               baseBMP ${((1 - bbmp.length/deflateB64.length)*100).toFixed(1)}% smaller`);
   console.log(`  Ratio:       ${(text.length/bbmp.length).toFixed(2)}x (baseBMP)`);
@@ -282,6 +293,7 @@ const runs = 10;
 for (const [label, fn] of [
   ['Deflate+base64url', () => { const c = zlib.deflateSync(Buffer.from(timingText), { level: 9 }); Buffer.from(c).toString('base64url'); }],
   ['ARX+base76', () => { const s = dictEncode(timingText); const c = brotli(new TextEncoder().encode(s)); encBase76(c); }],
+  ['ARX+base64url', () => { const s = dictEncode(timingText); const c = brotli(new TextEncoder().encode(s)); encBase64url(c); }],
   ['ARX+base1k', () => { const s = dictEncode(timingText); const c = brotli(new TextEncoder().encode(s)); encBase1k(c); }],
   ['ARX+baseBMP', () => { const s = dictEncode(timingText); const c = brotli(new TextEncoder().encode(s)); encBaseBMP(c); }],
 ]) {

skills/agent-render-linking/SKILL.md

@@ -33,7 +33,7 @@ Supported codecs:
 - `plain`: base64url-encoded JSON envelope
 - `lz`: `lz-string` compressed JSON encoded for URL-safe transport
 - `deflate`: deflate-compressed UTF-8 JSON bytes encoded as base64url
-- `arx`: domain-dictionary substitution + brotli (quality 11) + base76/base1k/baseBMP encoding (~70% smaller than deflate with baseBMP). Fetch the shared dictionary from `https://agent-render.com/arx-dictionary.json` to apply substitutions locally before brotli compression. Three encoding tiers: baseBMP (~62k safe BMP code points, ~15.92 bits/char, best density), base1k (1774 Unicode code points U+00A1–U+07FF), and base76 (ASCII fallback). The encoder tries all three and picks the shortest.
+- `arx`: domain-dictionary substitution + brotli (quality 11) + binary-to-text encoding (~70% smaller than deflate with baseBMP). Fetch the shared dictionary from `https://agent-render.com/arx-dictionary.json` to apply substitutions locally before brotli compression. Four wire shapes: baseBMP (~62k safe BMP code points, ~15.92 bits/char, best raw density), base1k (1774 Unicode code points U+00A1–U+07FF), base64url (ASCII `A-Za-z0-9-_`, `B.` prefix — good when Unicode would be percent-encoded), and base76 (77-char ASCII). The product encoder tries all four and picks the shortest **transport** length.
 - packed wire mode (`p: 1`) may be used automatically to shorten transport keys
 
 Prefer:
@@ -187,11 +187,11 @@ To use the dictionary for local `arx` encoding:
 1. Fetch `https://agent-render.com/arx-dictionary.json`
 2. Apply substitutions in order: for each entry, replace all occurrences of the pattern in the serialized JSON envelope with its corresponding control byte(s)
 3. Brotli-compress the substituted bytes at quality 11
-4. Encode the compressed bytes using **baseBMP** (preferred, smallest), **base1k** (mid-tier), or **base76** (ASCII fallback)
+4. Encode the compressed bytes; try **baseBMP**, **base1k**, **base64url**, and **base76**, then pick the shortest transport representation
     - BaseBMP uses ~62k safe BMP code points (U+00A1–U+FFEF, skipping surrogates, combining marks, zero-width chars). Prefix the encoded string with U+FFF0 marker. ~15.92 bits/char
     - Base1k uses 1774 Unicode code points (U+00A1–U+07FF, skipping combining diacriticals and soft hyphen). ~10.79 bits/char
-    - Base76 uses 77 ASCII fragment-safe characters — use this if the target surface cannot handle Unicode in URL fragments. ~6.27 bits/char
-    - Try all three and pick the shortest
+    - Base64url: standard `A-Za-z0-9-_` (no padding), prefix `B.` — ASCII-only, survives Discord/Slack/Teams-style handling better than Unicode-heavy fragments
+    - Base76 uses 77 ASCII fragment-safe characters. ~6.27 bits/char
 5. Prepend `v1.arx.<dictVersion>.` to form the fragment payload (use the same dictionary version used for substitution)
 
 The dictionary includes JSON envelope boilerplate patterns (like `","kind":"Markdown","content":"`), JSON-escaped Markdown syntax, programming keywords, and common English words. The viewer loads the same dictionary on startup to reverse substitutions during decode.

src/lib/payload/arx-codec.ts

@@ -1,7 +1,8 @@
 /**
  * arx codec — Agent Render eXtreme compression
  *
- * Pipeline:  text → dictionary substitution → brotli (quality 11) → base76 URL-fragment-safe encoding
+ * Pipeline:  text → dictionary substitution → brotli (quality 11) → binary-to-text encoding
+ * (base76, base1k, baseBMP, or base64url with a `B.` wire prefix)
  *
  * Achieves ~26% smaller fragments than deflate+base64url on typical payloads by combining:
  *   1. Dictionary substitution: replaces common multi-char patterns with short control bytes
@@ -567,11 +568,66 @@ export function decodeBaseBMP(str: string): Uint8Array {
   return result;
 }
 
-/** Returns true if the encoded string uses baseBMP encoding (starts with U+FFEE marker). */
+/** Returns true if the encoded string uses baseBMP encoding (starts with U+FFF0 marker). */
 export function isBaseBMPEncoded(str: string): boolean {
   return str.startsWith(BMP_MARKER);
 }
 
+// ---------------------------------------------------------------------------
+// Base64url — ASCII-only, chat/URL-safe (Discord, Slack, Teams)
+//
+// Standard RFC 4648 base64url alphabet (A-Za-z0-9-_) with no padding.
+// Wire prefix `B.` distinguishes this layer from base76 (length prefix),
+// base1k, and baseBMP: base76 can also begin with `B.` for some byte lengths,
+// so {@link arxDecompress} tries base64url first and falls back to base76
+// when Brotli decompression fails.
+// ---------------------------------------------------------------------------
+
+const BASE64URL_WIRE_PREFIX = "B.";
+
+function uint8ArrayToBase64Url(bytes: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  const base64 = btoa(binary);
+  return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
+}
+
+function base64UrlToUint8Array(input: string): Uint8Array {
+  const normalized = input.replace(/-/g, "+").replace(/_/g, "/");
+  const padding = normalized.length % 4 === 0 ? "" : "=".repeat(4 - (normalized.length % 4));
+  const binary = atob(`${normalized}${padding}`);
+  return Uint8Array.from(binary, (char) => char.charCodeAt(0));
+}
+
+/** Encodes bytes as base64url (no padding), prefixed with `B.` for ARX wire disambiguation. */
+export function encodeBase64url(bytes: Uint8Array): string {
+  if (bytes.length === 0) {
+    return BASE64URL_WIRE_PREFIX;
+  }
+  return BASE64URL_WIRE_PREFIX + uint8ArrayToBase64Url(bytes);
+}
+
+/** Decodes a string produced by {@link encodeBase64url} (requires the `B.` prefix). */
+export function decodeBase64url(str: string): Uint8Array {
+  if (!str.startsWith(BASE64URL_WIRE_PREFIX)) {
+    throw new Error("Expected base64url ARX payload with B. prefix.");
+  }
+  const body = str.slice(BASE64URL_WIRE_PREFIX.length);
+  if (body.length === 0) {
+    return new Uint8Array(0);
+  }
+  return base64UrlToUint8Array(body);
+}
+
+/** True when the payload uses the base64url ARX wire form (`B.` + optional base64url body). */
+export function isBase64urlEncoded(str: string): boolean {
+  if (!str.startsWith(BASE64URL_WIRE_PREFIX)) return false;
+  const rest = str.slice(BASE64URL_WIRE_PREFIX.length);
+  return /^[A-Za-z0-9_-]*$/.test(rest);
+}
+
 // ---------------------------------------------------------------------------
 // Brotli wrapper — lazy-loads brotli-wasm for browser compatibility
 // ---------------------------------------------------------------------------
@@ -626,14 +682,41 @@ export async function arxCompressBMP(json: string): Promise<string> {
   return encodeBaseBMP(compressed);
 }
 
+/**
+ * Compress with the arx pipeline using base64url for the binary-to-text step.
+ * ASCII-only and safe on surfaces that percent-encode non-ASCII (unlike base1k/baseBMP).
+ */
+export async function arxCompressBase64url(json: string): Promise<string> {
+  const brotli = await getBrotli();
+  const substituted = dictEncode(json);
+  const compressed = brotli.compress(new TextEncoder().encode(substituted), { quality: 11 });
+  return encodeBase64url(compressed);
+}
+
 /** Public API for `arxDecompress`. */
 export async function arxDecompress(encoded: string): Promise<string> {
   const brotli = await getBrotli();
-  const bytes = isBaseBMPEncoded(encoded)
-    ? decodeBaseBMP(encoded)
-    : isBase1kEncoded(encoded)
-      ? decodeBase1k(encoded)
-      : decodeBase76(encoded);
-  const decompressed = brotli.decompress(bytes);
-  return dictDecode(new TextDecoder().decode(decompressed));
+
+  const decompressFromBytes = (bytes: Uint8Array): string => {
+    const out = brotli.decompress(bytes);
+    return dictDecode(new TextDecoder().decode(out));
+  };
+
+  if (isBaseBMPEncoded(encoded)) {
+    return decompressFromBytes(decodeBaseBMP(encoded));
+  }
+
+  if (isBase64urlEncoded(encoded)) {
+    try {
+      return decompressFromBytes(decodeBase64url(encoded));
+    } catch {
+      // base76 length prefix can also be `B.` (e.g. 140-byte payloads); retry as base76.
+    }
+  }
+
+  if (isBase1kEncoded(encoded)) {
+    return decompressFromBytes(decodeBase1k(encoded));
+  }
+
+  return decompressFromBytes(decodeBase76(encoded));
 }

src/lib/payload/fragment.ts

@@ -2,7 +2,14 @@ import { compressToEncodedURIComponent, decompressFromEncodedURIComponent } from
 import { deflateSync, inflateSync, strFromU8, strToU8 } from "fflate";
 import { normalizeEnvelope } from "@/lib/payload/envelope";
 import { packEnvelope, unpackEnvelope } from "@/lib/payload/wire-format";
-import { arxCompress, arxCompressUnicode, arxCompressBMP, arxDecompress, getActiveDictVersion } from "@/lib/payload/arx-codec";
+import {
+  arxCompress,
+  arxCompressUnicode,
+  arxCompressBMP,
+  arxCompressBase64url,
+  arxDecompress,
+  getActiveDictVersion,
+} from "@/lib/payload/arx-codec";
 import {
   codecs,
   MAX_DECODED_PAYLOAD_LENGTH,
@@ -161,16 +168,17 @@ async function buildArxCandidates(envelope: PayloadEnvelope, packed: boolean): P
   const payloadEnvelope = { ...envelope, codec: "arx" as PayloadCodec };
   const json = JSON.stringify(packed ? packEnvelope(payloadEnvelope) : payloadEnvelope);
   const dictVersion = getActiveDictVersion();
-  const [ascii, unicode, bmp] = await Promise.all([
+  const [ascii, unicode, bmp, b64url] = await Promise.all([
     arxCompress(json),
     arxCompressUnicode(json),
     arxCompressBMP(json),
+    arxCompressBase64url(json),
   ]);
   const makeCandidate = (payload: string): CandidateFragment => {
     const value = `${PAYLOAD_FRAGMENT_KEY}=v1.arx.${dictVersion}.${payload}`;
     return { value, codec: "arx", packed, transportLength: computeTransportLength(value) };
   };
-  return [makeCandidate(ascii), makeCandidate(unicode), makeCandidate(bmp)];
+  return [makeCandidate(ascii), makeCandidate(unicode), makeCandidate(bmp), makeCandidate(b64url)];
 }
 
 async function buildCandidatesAsync(envelope: PayloadEnvelope, options: EncodeOptions): Promise<CandidateFragment[]> {