From 248879747d9b0cce2aba8e54e746f190ff54caab Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:16:33 -0500 Subject: [PATCH 1/9] redaction: one shared list of token shapes, Discord and Tailscale included (#889) A Discord bot token passed through redactString unredacted: its pattern lived only in src/watchdog/incidents.ts, so the event log, comments, chat notices and the dashboard's service log kept it. Tailscale keys had the same split in src/tailscale-setup.ts. Both patterns move into PATTERNS in src/events/redact.ts, the one list of credential shapes; incidents.ts, the watchdog's posts and the Tailscale step lines call redactString. The dashboard's service-log file is redacted as one text (with a margin above the kept lines) before it is split, so a private key over several lines is found whole. A bare 32-character key with no prefix stays uncovered by decision: a commit hash, a digest or an engine id has the same shape. test/redact-shapes.test.ts drives a stand-in for every shape (GitHub, Anthropic, Discord, Slack, Linear, Tailscale and the rest) through redactString and each output path: events, comments (file and GitHub backlogs), the superfan digest, notifications (incident post, merge post, chat notice) and the dashboard's service log (journal and file). The backstop, test/redaction-one-module.test.ts, fails on a replace/split in src/, web/src/ or scripts/ whose regex takes in a token stand-in, outside src/events/redact.ts; three own redactors broader than the shared shapes are listed in KNOWN_OWN_REDACTORS (never-grow). PATTERNS, the stand-ins and the lint's fixtures are registered in the weakening hold. Test (fails without): test/redact-shapes.test.ts and test/redaction-one-module.test.ts, 13 of 15 failing on origin/main (Discord and Tailscale in every path, the PEM in the log file, and the lint naming incidents.ts and tailscale-setup.ts). Signed-off-by: Jayden Vasquez --- src/dashboard.ts | 9 +- src/events/redact.ts | 23 +++- src/merge-policy-weakening.ts | 8 ++ src/tailscale-setup.ts | 8 +- src/watchdog/incidents.ts | 10 +- src/watchdog/post.ts | 5 +- test/redact-shapes.test.ts | 177 +++++++++++++++++++++++++++++ test/redaction-one-module.test.ts | 181 ++++++++++++++++++++++++++++++ test/tailscale-setup.test.ts | 2 +- test/token-standins.ts | 97 ++++++++++++++++ 10 files changed, 505 insertions(+), 15 deletions(-) create mode 100644 test/redact-shapes.test.ts create mode 100644 test/redaction-one-module.test.ts create mode 100644 test/token-standins.ts diff --git a/src/dashboard.ts b/src/dashboard.ts index 318aff4d1..92c71b3f9 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -884,6 +884,8 @@ const journalctl: JournalReader = (unit, lines) => { }; const clip = (s: string) => redactString(s.length > 2000 ? `${s.slice(0, 2000)} …` : s); +/** Lines read above a service log file's kept lines, so a private key that starts there is redacted whole (a 4096-bit key is about 50 lines). */ +const KEY_MARGIN = 100; /** * The coordinator's service log, read-only, as the dashboard's user (the @@ -917,7 +919,12 @@ export function readServiceLog(src: LogSource, lines = 200, read: JournalReader } if (src.file && existsSync(src.file)) { const text = readFileSync(src.file, 'utf8'); - const entries = text.split('\n').filter((l) => l.trim()).slice(-lines).map((message) => ({ at: null, priority: null, message: clip(message) })); + // The last lines, and KEY_MARGIN more above them, are redacted as one text before they are split, so a secret over + // several lines (a private key, even one that starts above the kept lines) is found whole (#889); each line is + // redacted again as it is clipped. + const tail = text.split('\n').filter((l) => l.trim()).slice(-(lines + KEY_MARGIN)); + const kept = redactString(tail.join('\n')).split('\n').filter((l) => l.trim()).slice(-lines); + const entries = kept.map((message) => ({ at: null, priority: null, message: clip(message) })); return { source: 'file', unit: src.unit, entries, problem: entries.length ? null : problem }; } return { source: 'none', unit: src.unit, entries: [], problem: problem ?? 'no service log found: the coordinator runs in the foreground or as a service without a log file' }; diff --git a/src/events/redact.ts b/src/events/redact.ts index 59d8d3fe9..0212f4c06 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -2,6 +2,8 @@ // based; gitleaks still scans transcripts. False positives cost a few // characters of a log line; a false negative leaks a key into the log. // This is the one shared redactor: a feature that needs more redaction adds a pattern here, never its own redactor. +// PATTERNS is the one list of credential shapes (#889); test/redaction-one-module.test.ts fails on a module that +// redacts a token shape with its own regex, and test/redact-shapes.test.ts drives every shape through every output path. // // Every pattern added here must run in linear time on hostile input (an intake report, a CI log, chat text): a // match may start only where a run of its characters starts (a prefix group, never `\b` inside a run of `-`), and a @@ -63,6 +65,9 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // Linear personal API keys and OAuth tokens. [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}\b/g, 'lin_[REDACTED]', 'Linear key'], [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], + // Tailscale auth keys, API tokens and OAuth client secrets (tskey-auth-..., tskey-api-..., tskey-client-...): the + // prefix and the rest of its run, however short (src/tailscale-setup.ts redacted them this way on its own). + [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]', 'private key'], [/\b(Bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], // Credentials inside connection strings: keep scheme, user and host. @@ -79,7 +84,15 @@ const PATTERNS: [RegExp, Replacement, string][] = [ (m, q, k, sep, v) => (credentialKey(k!) && credentialShaped(v!) ? `${q}${k}${q}${sep}${q}[REDACTED]${q}` : m), 'secret under a JSON key', ], + // A Discord bot token: three base64url parts (the bot's id, a timestamp, an HMAC). Last, as src/watchdog/incidents.ts + // applied it after the rest (#889: it was redacted only there; there it started at `\b`, here after any character + // but a letter or digit, so `x_` is found too). It may start inside a run of `-` or `_`, but the first part + // is bounded (23 to 28 characters, then a dot), so each start reads a bounded stretch: linear (redact-shapes.test.ts). + [/(^|[^A-Za-z0-9])[A-Za-z0-9_-]{23,28}\.[A-Za-z0-9_-]{6,7}\.[A-Za-z0-9_-]{27,}\b/g, '$1[REDACTED]', 'Discord bot token'], ]; +// Not covered, by decision (#889): a bare 32-character key with no prefix, such as a dashboard key standing alone. A +// commit hash, a digest and the engine's own ids (randomBytes(16) in hex) have the same shape. A key in a Bearer +// header, a JSON value under a credential key, or a URL's password is covered; a key is never put in a URL. /** * The first word of a dotted name that is code, not a host: `this.internal`, `opts.corp`, `config.lan.x`. Only @@ -142,8 +155,14 @@ const IDENTIFIERS: [RegExp, Replacement, string][] = [ [/(^|[^\w:])f(?:[cd][0-9a-f]{2}|e[89ab][0-9a-f]):(?:[0-9a-f]{0,4}:){1,6}[0-9a-f]{0,4}(?![\w:])/gi, '$1[private-ip]', 'private IP'], ]; -/** Every pattern with its kind, for the linear-time test (test/redact.test.ts). */ -export const REDACTION_PATTERNS: readonly { kind: string; re: RegExp; rep: Replacement }[] = [...PATTERNS, ...IDENTIFIERS].map(([re, rep, kind]) => ({ kind, re, rep })); +/** + * Every pattern with its kind, for the linear-time test (test/redact.test.ts) and the shape tests + * (test/redact-shapes.test.ts): `secret` for a secret's pattern (applied by default), not for an identifier's. + */ +export const REDACTION_PATTERNS: readonly { kind: string; re: RegExp; rep: Replacement; secret: boolean }[] = [ + ...PATTERNS.map(([re, rep, kind]) => ({ kind, re, rep, secret: true })), + ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, secret: false })), +]; export interface RedactTextOptions { /** Also redact emails, internal hostnames, private IPs and home paths' user names (text for an issue, comment or PR). */ diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 28723fd9e..bcd91460b 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -62,6 +62,9 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'test/owner-terminal-rerun.test.ts', table: 'FIRST_TIME_ONLY', kind: 'never-grow' }, { path: 'scripts/lib/denylist.mjs', table: 'SKIP', kind: 'never-grow' }, { path: 'src/guardrails/engine.ts', table: 'AGENT_EDITABLE', kind: 'never-grow' }, + // #889: the modules that still redact a token shape on their own, and the one-module lint's good fixtures. + { path: 'test/redaction-one-module.test.ts', table: 'KNOWN_OWN_REDACTORS', kind: 'never-grow' }, + { path: 'test/redaction-one-module.test.ts', table: 'GOOD_SOURCES', kind: 'never-grow' }, // The lints' rules and bad fixtures, and the guards' lists: taking one out is a loosening. { path: 'test/change-files-renames.test.ts', table: 'READER_WORDS', kind: 'never-shrink' }, { path: 'test/review.test.ts', table: 'LOCK_SCOPE_BAD', kind: 'never-shrink' }, @@ -80,6 +83,10 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'test/machine-path-writes.test.ts', table: 'GUARD', kind: 'never-shrink' }, // The golden runner settings (#856): the deny list and hooks a run's --settings must carry. { path: 'test/runner-golden.test.ts', table: 'GUARD', kind: 'never-shrink' }, + // #889: the shared redactor's one list of credential shapes, a stand-in for each, and the one-module lint's bad fixtures. + { path: 'src/events/redact.ts', table: 'PATTERNS', kind: 'never-shrink' }, + { path: 'test/token-standins.ts', table: 'SHAPES', kind: 'never-shrink' }, + { path: 'test/redaction-one-module.test.ts', table: 'BAD_SOURCES', kind: 'never-shrink' }, // The guards' own tables (their files are guard files too; the table names what was taken out). { path: 'src/review.ts', table: 'SELF_MODIFICATION_PARTS', kind: 'never-shrink' }, { path: 'src/review.ts', table: 'LOCKED_CATEGORIES', kind: 'never-shrink' }, @@ -124,6 +131,7 @@ export const LINT_TESTS: readonly string[] = [ 'test/web-buttons.test.ts', 'test/superfan.test.ts', 'test/claude-accounts-retired.test.ts', + 'test/redaction-one-module.test.ts', ]; /** diff --git a/src/tailscale-setup.ts b/src/tailscale-setup.ts index 879e37a86..4e3bd4a71 100644 --- a/src/tailscale-setup.ts +++ b/src/tailscale-setup.ts @@ -20,6 +20,7 @@ import { readFileSync, statSync } from 'node:fs'; import { join } from 'node:path'; import { BRAND } from './brand.js'; import type { EventLog } from './events/log.js'; +import { redactString } from './events/redact.js'; import type { StoredEvent } from './events/types.js'; import type { Role } from './console/asks.js'; import { isMachineOwner, MACHINE_OWNERS_FILE, machineChangeRefusal, machineOwners, type MachineRequester } from './machine-owners.js'; @@ -112,8 +113,11 @@ export interface StepLine { detail: string; } -/** Take a link or a key out of a line before it is shown: the script never writes one, and this is the check on that. */ -const scrub = (s: string) => s.replace(/https?:\/\/login\.tailscale\.com\S*/g, '[the login link]').replace(/tskey-[A-Za-z0-9_-]+/g, '[a key]'); +/** + * Take a key or a link out of a line before it is shown: the script never writes one, and this is the check on that. + * A key (and any other secret) by the shared redactor (#889), then the login link. + */ +const scrub = (s: string) => redactString(s).replace(/https?:\/\/login\.tailscale\.com\S*/g, '[the login link]'); /** Which version of the progress file this is (its size and change time; '' when there is none): it changes with every line. */ export function stepsStamp(dir = TAILSCALE_DIR): string { diff --git a/src/watchdog/incidents.ts b/src/watchdog/incidents.ts index aa2ec7347..0c47c6194 100644 --- a/src/watchdog/incidents.ts +++ b/src/watchdog/incidents.ts @@ -127,14 +127,10 @@ export interface RaiseOptions { /** The store's directory: the machine's slot directory, which every instance's coordinator can write. */ export const incidentsDir = (dir?: SlotPathLike): SlotPathLike => dir ?? slotsDir(); -/** A Discord bot token's shape (three dot-separated base64url parts): never stored or posted, whoever's it is. */ -export const DISCORD_TOKEN = /\b[A-Za-z0-9_-]{23,28}\.[A-Za-z0-9_-]{6,7}\.[A-Za-z0-9_-]{27,}\b/g; - -/** A text with every secret-shaped string taken out. */ -export const redactIncidentText = (s: string) => redactString(s).replace(DISCORD_TOKEN, '[REDACTED]'); - +// Every secret-shaped string is taken out by the shared redactor, a Discord bot token's among them (#889: its pattern +// was here, and only here). const clip = (s: string, n: number) => { - const r = redactIncidentText(String(s ?? '')).replace(/\s+/g, ' ').trim(); + const r = redactString(String(s ?? '')).replace(/\s+/g, ' ').trim(); return r.length > n ? `${r.slice(0, n - 1)}…` : r; }; diff --git a/src/watchdog/post.ts b/src/watchdog/post.ts index 58cf90210..5f001e4e7 100644 --- a/src/watchdog/post.ts +++ b/src/watchdog/post.ts @@ -2,7 +2,8 @@ // its place on the Health page. Machine-wide at most RATE.max posts per RATE.windowMs; the rest wait for the next // pass, oldest first. An incident resolved before its first post was ever sent is not posted at all (it came and // went inside the limit). Every post is redacted again here, and the bot's own token is never in one. -import { readIncidents, redactIncidentText, withIncidents, type Incident } from './incidents.js'; +import { readIncidents, withIncidents, type Incident } from './incidents.js'; +import { redactString } from '../events/redact.js'; import type { SlotPathLike } from '../slot-file.js'; /** At most this many posts per window, machine-wide. */ @@ -15,7 +16,7 @@ export type Poster = (content: string) => Promise; /** A text with every secret-shaped string, and each of `secrets` (the bot token), taken out. */ export function scrub(text: string, secrets: string[] = []): string { - let out = redactIncidentText(text); + let out = redactString(text); for (const s of secrets) if (s && s.length >= 8) out = out.split(s).join('[REDACTED]'); return out; } diff --git a/test/redact-shapes.test.ts b/test/redact-shapes.test.ts new file mode 100644 index 000000000..cf6f56260 --- /dev/null +++ b/test/redact-shapes.test.ts @@ -0,0 +1,177 @@ +// #889: one shared redactor (src/events/redact.ts) knows every credential shape the engine handles, and every output +// path redacts through it. Each stand-in in test/token-standins.ts goes through redactString, and through each path +// that stores or sends text: the event log, issue comments (file and GitHub backlogs), the superfan's digest, +// the notifications (an incident post, a merge post, a chat notice) and the dashboard's service-log view. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { redactString, REDACTION_PATTERNS, secretKinds } from '../src/events/redact.js'; +import { EventLog } from '../src/events/log.js'; +import { FileBacklog } from '../src/backlog/file.js'; +import { GitHubBacklog } from '../src/backlog/github.js'; +import { budgetFor } from '../src/github/budget.js'; +import { buildDigest } from '../src/superfan/digest.js'; +import { redactPii } from '../src/intake/redact.js'; +import { postText, scrub } from '../src/watchdog/post.js'; +import type { Incident } from '../src/watchdog/incidents.js'; +import { mergePostText } from '../src/merge-posts.js'; +import { mainWatchPost } from '../src/intake/channel.js'; +import { readServiceLog } from '../src/dashboard.js'; +import { leaks, run, SHAPES, type StandIn } from './token-standins.js'; + +const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); +/** Each stand-in in a log line, and as an environment assignment. */ +const lines = (s: StandIn) => [`deploy failed: ${s.value} (exit 1)`, `error: env had TOKEN=${s.value}; giving up`]; +/** The stand-ins whose secret part survives `out(s)`, every miss listed. */ +const survivors = async (out: (text: string) => string | Promise) => { + const missed: string[] = []; + for (const s of SHAPES) for (const l of lines(s)) if (leaks(await out(l), s.secret)) missed.push(`${s.what} in "${l.slice(0, 18)}…"`); + return missed; +}; +/** The production composition the coordinator hands the digest and the merge posts (src/coordinator.ts, src/coordinator/superfan.ts). */ +const coordinatorRedact = (s: string) => scrub(redactPii(s, { names: [], patterns: [] })); + +test('every secret pattern of the shared redactor has a stand-in, and secretKinds names each stand-in by its kind', () => { + const kinds = [...new Set(REDACTION_PATTERNS.filter((p) => p.secret).map((p) => p.kind))]; + assert.deepEqual(kinds.filter((k) => !SHAPES.some((s) => s.kind === k)), [], 'add a stand-in to test/token-standins.ts for each new pattern'); + assert.deepEqual(SHAPES.filter((s) => !secretKinds(s.value).includes(s.kind)).map((s) => s.what), []); + for (const k of ['GitHub token', 'API key (sk-)', 'Discord bot token', 'Slack token', 'Slack app-level token', 'Slack webhook', 'Linear key', 'Tailscale key']) { + assert.ok(kinds.includes(k), `the shared redactor knows ${k}`); + } +}); + +test('redactString: no stand-in survives, Discord included', async () => { + assert.deepEqual(await survivors((t) => redactString(t)), []); +}); + +test('redactString: a Discord bot token right after an underscore, a hyphen, an = or a quote is redacted too', () => { + const { value } = SHAPES.find((s) => s.kind === 'Discord bot token')!; + for (const before of ['x_', 'x-', 'DISCORD_TOKEN=', '"', 'Bot ']) assert.equal(redactString(`${before}${value} ok`), `${before}[REDACTED] ok`); +}); + +test('events: no stand-in survives into the event log', async () => { + const log = new EventLog(':memory:'); + assert.deepEqual( + await survivors((t) => { + const e = log.append('coordinator.error', { instance: 'alpha', where: 'deploy', kind: 'error', message: t }, 'coordinator'); + return JSON.stringify([e.payload, log.read(e.id - 1)]); + }), + [], + ); +}); + +test('comments: no stand-in survives into an issue comment on the file backlog', async () => { + const f = new FileBacklog(join(scratch('bl-'), 'b.json'), 'harness'); + const n = f.open({ title: 'x', body: '', author: 'a' }); + assert.deepEqual( + await survivors(async (t) => { + await f.comment(n, t); + return JSON.stringify((await f.comments(n)).at(-1)); + }), + [], + ); +}); + +test('comments: no stand-in survives into an issue comment sent to GitHub', async () => { + const sent: string[] = []; + const capture = (async (_url: string, init?: RequestInit) => { + sent.push(String(init?.body ?? '')); + return new Response(JSON.stringify({ id: 1 }), { status: 200 }); + }) as typeof fetch; + const slots = scratch('slots-'); + const gh = new GitHubBacklog('o/r', () => 't', capture, undefined, (tk) => budgetFor(tk, slots)); + assert.deepEqual( + await survivors(async (t) => { + await gh.comment(1, t); + return sent.at(-1)!; + }), + [], + ); +}); + +test("digest: no stand-in survives into the superfan's digest", async () => { + const cursor = { since: '2026-01-01T00:00:00.000Z', eventId: 0, open: null, health: null }; + assert.deepEqual( + await survivors((t) => { + const d = buildDigest({ journal: { entries: [{ at: '2026-02-01T00:00:00.000Z', priority: 3, message: t }], problem: null } }, cursor, new Date('2026-02-02T00:00:00.000Z'), coordinatorRedact); + assert.equal(d.items.length, 1); + return d.items[0]!.line; + }), + [], + ); +}); + +test("notifications: no stand-in survives into an incident's ops-channel post", async () => { + const incident = (detail: string): Incident => ({ id: 'i1', kind: 'k', key: 'k', instance: 'alpha', severity: 'red', title: 'Deploy failed', detail, fix: 'Rotate it.', tried: [], since: '2026-02-01T00:00:00.000Z', lastSeen: '2026-02-01T00:00:00.000Z', resolvedAt: null, posted: {} }); + assert.deepEqual(await survivors((t) => postText(incident(t), 'raised', { dashboardUrl: 'https://dash.example.test' })), []); +}); + +test('notifications: no stand-in survives into a merge post', async () => { + const pr = (body: string) => ({ number: 7, url: 'https://github.com/o/r/pull/7', title: 'Fix it', body, mergedAt: null, mergeCommit: null, mergedBy: 'ada', base: 'main' }); + assert.deepEqual(await survivors((t) => mergePostText(pr(t), 'a3f1c9e0b2d4', coordinatorRedact)), []); +}); + +test("notifications: no stand-in survives into a chat notice (the main watch's alert)", async () => { + const notice = (reason: string) => mainWatchPost({ sha: 'a3f1c9e0b2d4', url: 'https://github.com/o/r/commit/a3f1c9e0b2d4', reason, revert: null, no_revert: null }, { dashboardUrl: 'https://dash.example.test' }).text; + assert.deepEqual(await survivors(notice), []); +}); + +test("dashboard: no stand-in survives into the service-log view from the journal", async () => { + assert.deepEqual( + await survivors((t) => { + const journal = () => ({ status: 0, stdout: `${JSON.stringify({ MESSAGE: t, PRIORITY: '3', __REALTIME_TIMESTAMP: '1767225600000000' })}\n`, stderr: '' }); + return JSON.stringify(readServiceLog({ unit: 'coordinator.service', file: null }, 10, journal).entries); + }), + [], + ); +}); + +test("dashboard: no stand-in survives into the service-log view from the log file, a private key over several lines included", async () => { + const dir = scratch('svc-log-'); + assert.deepEqual( + await survivors((t) => { + const file = join(dir, 'coordinator.log'); + writeFileSync(file, `${t}\n`); + return JSON.stringify(readServiceLog({ unit: null, file }, 10).entries); + }), + [], + ); +}); + +test('dashboard: a private key that starts above the kept lines of the log file is redacted whole', () => { + const pem = SHAPES.find((s) => s.kind === 'private key')!; + const file = join(scratch('svc-log-'), 'coordinator.log'); + writeFileSync(file, `${pem.value}\n${Array.from({ length: 8 }, (_, i) => `line ${i}`).join('\n')}\n`); + const shown = readServiceLog({ unit: null, file }, 10).entries; + assert.ok(!leaks(JSON.stringify(shown), pem.secret), JSON.stringify(shown)); + assert.equal(shown.at(-1)!.message, 'line 7'); +}); + +test('not covered, by decision: a bare 32-character key with no prefix (a commit hash, a digest or an id looks the same); in a Bearer header it is redacted', () => { + const bare = run(32); + const hex = 'a3f1c9e0b2d4f6a8'.repeat(2); + for (const v of [bare, hex]) assert.equal(redactString(`key ${v} rotated`), `key ${v} rotated`); + assert.ok(!redactString(`Authorization: Bearer ${bare}`).includes(bare)); +}); + +test('Discord and Tailscale patterns run in linear time on hostile input', () => { + const N = 100_000; + const fill = (unit: string) => unit.repeat(Math.ceil(N / unit.length)); + const hostile = [fill('a-'), fill('a.'), fill(`${'a'.repeat(25)}.`), fill(`${'a'.repeat(25)}.bbbbbb.`), fill(`${'a'.repeat(25)}.bbbbbb.-c`), fill('-a'), fill(`${'a-'.repeat(14)}.bbbbbb.${'c-'.repeat(40)}.`), fill('tskey-'), fill('tskey'), fill('tskey-a-')]; + const slow: string[] = []; + const patterns = REDACTION_PATTERNS.filter((p) => p.kind === 'Discord bot token' || p.kind === 'Tailscale key'); + assert.equal(patterns.length, 2, 'both patterns are in the shared redactor'); + for (const { kind, re, rep } of patterns) { + for (const s of hostile) { + const t0 = performance.now(); + if (typeof rep === 'string') s.replace(re, rep); + else s.replace(re, rep); + const ms = performance.now() - t0; + if (ms > 300) slow.push(`${kind} on ${s.slice(0, 12)}…: ${Math.round(ms)} ms`); + } + } + assert.deepEqual(slow, []); +}); diff --git a/test/redaction-one-module.test.ts b/test/redaction-one-module.test.ts new file mode 100644 index 000000000..694604ef3 --- /dev/null +++ b/test/redaction-one-module.test.ts @@ -0,0 +1,181 @@ +// #889, the backstop: no module redacts a credential with its own regex. The protection is by design: the one list of +// shapes is PATTERNS in src/events/redact.ts, every output path calls redactString or redact (test/redact-shapes.test.ts +// drives each), and a module that needs a shape adds it there. A second list drifts: a Discord bot token was redacted +// only in src/watchdog/incidents.ts, and passed through redactString everywhere else. +// +// The lint: every `replace`, `replaceAll` or `split` call (by `.`, `?.` or `['...']`) in src/, web/src/ and scripts/ (TypeScript and JavaScript) +// whose pattern is a regex literal, a `new RegExp('...')` with a literal source, or a name bound to one (in the file, +// or else any file's const of that name), is run on each token stand-in (test/token-standins.ts). A match that +// takes in a whole token, and nothing of the words around it, is an own redactor for that shape. +// Not caught: a pattern built at run time (a template with a substitution, String.raw, a concatenation, a function's +// result), a regex passed through a parameter or kept in an object's property, a match found and then cut out by +// string (match, then split and join), string methods other than these three, shell and other languages +// (scripts/setup/*.sh redact with sed, and a script's embedded JavaScript), and a shape no stand-in has (the coverage +// test in redact-shapes.test.ts asks for one per pattern). +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import ts from 'typescript'; +import { BRAND } from '../src/brand.js'; +import { repoRoot } from './helpers.js'; +import { SHAPES } from './token-standins.js'; + +/** The one module that holds the shapes. */ +const SHARED = 'src/events/redact.ts'; +const ROOTS = ['src', 'web/src', 'scripts']; +const CODE = /\.(?:ts|tsx|mts|cts|js|mjs|cjs)$/; +const CALLS = new Set(['replace', 'replaceAll', 'split']); +const BEFORE = 'deploy failed:'; +const AFTER = '(exit 1)'; + +/** + * Own redactors found today and not moved by #889, each with why: still found (so the list only shrinks), and no other + * one passes. Adding an entry is a loosening (registered never-grow). + */ +const KNOWN_OWN_REDACTORS: readonly { path: string; why: string }[] = [ + { path: 'scripts/bench/cli.ts', why: "the bench's scrub: its TOKEN_SHAPE is broader than the shared shapes (a GitHub token from 16 characters, any sk-ant- or xox- run), for a token cut short; redactString alone would narrow it" }, + { path: `scripts/machine/${BRAND.cli}-update.cjs`, why: "the root updater's journal redact(): a standalone CommonJS script that loads no engine module; broader than the shared shapes (GitHub from 20 characters, sk-ant- from 10)" }, + { path: 'src/gh-login.ts', why: "safeLine masks a GitHub token from 8 characters (gh's output may cut one short), as [hidden]; redactString alone would narrow it" }, +]; + +/** A regex from a literal's text (`/a+/gi`), or null when it doesn't compile here. */ +function literal(text: string): RegExp | null { + const end = text.lastIndexOf('/'); + try { + return new RegExp(text.slice(1, end), text.slice(end + 1)); + } catch { + return null; + } +} + +/** The regex a pattern expression is, when the lint can tell without running the code. */ +function regexOf(e: ts.Expression, bound: Map, global: Map): RegExp | null { + while (ts.isParenthesizedExpression(e) || ts.isAsExpression(e) || ts.isNonNullExpression(e)) e = e.expression; + if (ts.isRegularExpressionLiteral(e)) return literal(e.text); + if (ts.isNewExpression(e) || ts.isCallExpression(e)) { + const [src, flags] = e.arguments ?? []; + if (ts.isIdentifier(e.expression) && e.expression.text === 'RegExp' && src && ts.isStringLiteralLike(src)) { + try { + return new RegExp(src.text, flags && ts.isStringLiteralLike(flags) ? flags.text : ''); + } catch { + return null; + } + } + return null; + } + if (ts.isIdentifier(e)) return bound.get(e.text) ?? global.get(e.text) ?? null; + return null; +} + +const parse = (path: string, text: string) => ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, path.endsWith('.tsx') ? ts.ScriptKind.TSX : path.endsWith('.ts') || path.endsWith('.mts') || path.endsWith('.cts') ? ts.ScriptKind.TS : ts.ScriptKind.JS); + +/** The names a file binds to a regex it can read (`const SHAPE = /.../g`, or `const R = SHAPE` after it), top-level or not. */ +function bindings(sf: ts.SourceFile): Map { + const out = new Map(); + const visit = (n: ts.Node) => { + if (ts.isVariableDeclaration(n) && ts.isIdentifier(n.name) && n.initializer) { + const re = regexOf(n.initializer, out, new Map()); + if (re) out.set(n.name.text, re); + } + ts.forEachChild(n, visit); + }; + visit(sf); + return out; +} + +/** + * The token kinds `re` takes in: a match holds a stand-in's whole secret and none of the words around it, and `re` + * doesn't take in the ordinary part of the stand-in's control (ordinary text of its structure) the same way. + */ +function tokenKinds(re: RegExp): string[] { + const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`); + const takesIn = (text: string, part: string) => [...`${BEFORE} ${text} ${AFTER}`.matchAll(g)].some((m) => m[0].includes(part) && !m[0].includes(BEFORE) && !m[0].includes(AFTER)); + const kinds = new Set(); + for (const s of SHAPES) if (s.token && takesIn(s.value, s.secret) && !(s.control && takesIn(s.control.text, s.control.part))) kinds.add(s.kind); + return [...kinds].sort(); +} + +/** The own redactors in one file: `line: call(kinds)`, for each call whose pattern takes in a token. */ +export function ownRedactors(path: string, text: string, global: Map = new Map()): string[] { + const sf = parse(path, text); + const bound = bindings(sf); + const found: string[] = []; + const visit = (n: ts.Node) => { + // `s.replace(...)`, `s?.replace(...)` or `s['replace'](...)`. + const callee = ts.isCallExpression(n) ? n.expression : null; + const name = callee && ts.isPropertyAccessExpression(callee) ? callee.name.text : callee && ts.isElementAccessExpression(callee) && ts.isStringLiteralLike(callee.argumentExpression) ? callee.argumentExpression.text : null; + if (ts.isCallExpression(n) && name && CALLS.has(name) && n.arguments[0]) { + const re = regexOf(n.arguments[0], bound, global); + const kinds = re ? tokenKinds(re) : []; + if (kinds.length) found.push(`${sf.getLineAndCharacterOfPosition(n.getStart()).line + 1}: .${name}(${kinds.join(', ')})`); + } + ts.forEachChild(n, visit); + }; + visit(sf); + return found; +} + +function codeFiles(): string[] { + const out: string[] = []; + const walk = (d: string) => { + for (const e of readdirSync(join(repoRoot, d), { withFileTypes: true })) { + const p = `${d}/${e.name}`; + if (e.isDirectory()) { + if (e.name !== 'node_modules' && e.name !== 'fixtures' && e.name !== 'dist') walk(p); + } else if (CODE.test(e.name) && !e.name.endsWith('.d.ts')) out.push(p); + } + }; + for (const r of ROOTS) walk(r); + return out.sort(); +} + +/** Bad sources: each is an own redactor the lint must report. */ +const BAD_SOURCES: readonly string[] = [ + // incidents.ts's own Discord pattern, as it was. + "export const D = /\\b[A-Za-z0-9_-]{23,28}\\.[A-Za-z0-9_-]{6,7}\\.[A-Za-z0-9_-]{27,}\\b/g;\nexport const r = (s: string) => s.replace(D, '[REDACTED]');", + "const t = (s: string) => s.replace(/\\bgh[opsur]_\\w{16,}/g, 'x');", + "const t = (s: string) => s.replaceAll(new RegExp('xox[abprs]-[\\\\w-]+', 'g'), '');", + "const t = (s) => s.split(/lin_api_[A-Za-z0-9]+/).join('[redacted]');", + "const t = (s: string) => s.replace(/tskey-[A-Za-z0-9_-]+/g, '[a key]');", + "function f(s: string) { const K = /sk-ant-[\\w-]+/g; return s.replace(K, ''); }", + // An alias of a pattern, and a pattern in a template with no substitution. + "const SHAPE = /\\bxapp-[\\w-]+/g;\nconst R = SHAPE;\nexport const t = (s: string) => s.replace(R, '');", + 'const t = (s: string) => s.replace(new RegExp(`github_pat_\\\\w+`, "g"), "");', + // Called by element access. + "const t = (s: string) => s['replace'](/lin_api_\\w+/g, '');", + // A token pattern beside a link pattern is still a token pattern. + "const t = (s: string) => s.replace(/https?:\\/\\/\\S+|\\bghp_\\w+/g, '');", + "const t = (s: string) => s.replace(/https:\\/\\/hooks\\.slack\\.com\\/services\\/\\S+/g, '');", +]; +/** Good sources: none is an own redactor (the shared one called, a shape checked, text tidied). */ +const GOOD_SOURCES: readonly string[] = [ + "import { redactString } from './events/redact.js';\nconst t = (s: string) => redactString(s).replace(/\\s+/g, ' ');", + "const SHAPE = /^xoxb-[A-Za-z0-9-]{20,250}$/;\nconst ok = (s: string) => SHAPE.test(s);", + "const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');", + "const t = (s: string, k: string) => s.replaceAll(k, '[redacted]');", + "const t = (s: string) => s.replace(/https?:\\/\\/login\\.example\\.test\\S*/g, '[the login link]');", + // Every link or path handled alike (a webhook's among them) is link handling, not a token shape. + "const words = (s: string) => s.replace(/https?:\\/\\/\\S+/g, ' ').split(/(https?:\\/\\/[^\\s)]+)/g);", + "const cause = (s: string) => s.replace(/(?:[A-Za-z]:)?(?:[\\\\/][\\w.@~+-]+){2,}[\\\\/]?/g, '');", +]; + +test('the lint reports each bad source and no good one', () => { + assert.deepEqual(BAD_SOURCES.filter((s) => ownRedactors('fixture.ts', s).length === 0), []); + assert.deepEqual(GOOD_SOURCES.flatMap((s) => ownRedactors('fixture.ts', s)), []); +}); + +test('no module but src/events/redact.ts redacts a token shape with its own regex', () => { + const files = codeFiles(); + assert.ok(files.length > 300, `${files.length} files read`); + assert.ok(files.includes(SHARED)); + // Every file's regex names, for a pattern imported from another module (`import { SHAPE } from ...`). + const global = new Map(); + const texts = new Map(files.map((f) => [f, readFileSync(join(repoRoot, f), 'utf8')])); + for (const [f, text] of texts) if (f !== SHARED) for (const [k, v] of bindings(parse(f, text))) global.set(k, v); + const found = files.filter((f) => f !== SHARED).flatMap((f) => ownRedactors(f, texts.get(f)!, global).map((x) => `${f}:${x}`)); + const known = new Set(KNOWN_OWN_REDACTORS.map((k) => k.path)); + assert.deepEqual(found.filter((x) => !known.has(x.slice(0, x.indexOf(':')))), [], 'add the shape to PATTERNS in src/events/redact.ts and call redactString'); + for (const k of KNOWN_OWN_REDACTORS) assert.ok(found.some((x) => x.startsWith(`${k.path}:`)), `${k.path} no longer has its own redactor: take it out of KNOWN_OWN_REDACTORS`); +}); diff --git a/test/tailscale-setup.test.ts b/test/tailscale-setup.test.ts index 86dcc0dea..7eeca5879 100644 --- a/test/tailscale-setup.test.ts +++ b/test/tailscale-setup.test.ts @@ -158,7 +158,7 @@ test('the login link is read only when it is exactly Tailscale\'s login page, an writeFileSync(join(dir, 'steps.jsonl'), `${readFileSync(join(dir, 'steps.jsonl'), 'utf8')}{"at":"2026-10-13T00:00:00Z","step":"logout","state":"done","detail":"x"}\n{"at":"2026-10-13T0`); const lines = readStepLines(dir); assert.equal(lines.length, 2); - assert.equal(lines[1]!.detail, 'open [the login link] with [a key]'); + assert.equal(lines[1]!.detail, 'open [the login link] with tskey-[REDACTED]'); const run = stepRun(lines)!; assert.deepEqual([run.step, run.status], ['up', 'waiting']); assert.equal(stepRun(lines, Date.now() + 3600_000)!.status, 'stalled'); diff --git a/test/token-standins.ts b/test/token-standins.ts new file mode 100644 index 000000000..6e18833eb --- /dev/null +++ b/test/token-standins.ts @@ -0,0 +1,97 @@ +// Stand-ins for every credential shape the shared redactor (src/events/redact.ts) knows, one or more per kind (#889). +// Fake values in the real formats, assembled at run time so no token-shaped literal is committed. Used by +// test/redact-shapes.test.ts (each shape through each output path) and test/redaction-one-module.test.ts (no regex +// outside the shared redactor rewrites one of them). +// +// Each entry: the kind the shared redactor names it by (secretKinds), the text as it would appear, and the part of it +// that must never survive. `token` false: a structure around a value (a password in a URL, a value under a JSON key), +// not a token's own shape; the one-module lint reads only token shapes, since a broad rule elsewhere (an email +// pattern) may match a structure's text without being a token redactor. + +/** `seed` repeated and cut to `n` characters: mixed case with digits, like a random key. */ +export const run = (n: number, seed = 'A1b2C3d4E5f6G7h8') => seed.repeat(Math.ceil(n / seed.length)).slice(0, n); +const hex = (n: number) => '7e9b5d3c1f0a2468'.repeat(Math.ceil(n / 16)).slice(0, n); +const j = (sep: string, ...parts: string[]) => parts.join(sep); +const upper = (n: number) => 'Q7RZ4KX2M9TB'.repeat(Math.ceil(n / 12)).slice(0, n); + +export interface StandIn { + /** The kind src/events/redact.ts names it by (secretKinds). */ + kind: string; + /** A short name for the test output. */ + what: string; + /** The text as it would appear in a log line. */ + value: string; + /** What must not survive redaction (the secret part of `value`). */ + secret: string; + /** A token's own shape (true), or a structure around a value (false). */ + token: boolean; + /** + * Ordinary text of the same structure (a link, for a URL-shaped token), with an ordinary `part` where the secret is: + * a regex that takes that part in too handles the structure (every link, every path), not this token's shape. + */ + control?: { text: string; part: string }; +} + +const t = (kind: string, what: string, value: string, secret = value, control?: StandIn['control']): StandIn => ({ kind, what, value, secret, token: true, ...(control ? { control } : {}) }); + +const DISCORD = j('.', `MTAx${'MjM0NTY3ODkw'.repeat(2)}`, 'GhIjKl', `${'abcdefghij'.repeat(3)}AB`); +const DISCORD_NEW = j('.', `MTI${run(23, 'k5ODc2NTQzMjEw')}`, 'G7xQ_p', run(38)); +const PEM_BODY = run(64); +const PEM = j('\n', j(' ', '-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'), PEM_BODY, j(' ', '-----END', 'RSA', 'PRIVATE', 'KEY-----')); +const URL_PASSWORD = run(24, 'pW9xQ2zL'); +const JSON_VALUE = run(24, 'notArealValue7'); + +/** Every stand-in: a shape gone from here is a shape no test drives (registered never-shrink). */ +export const SHAPES: readonly StandIn[] = [ + // GitHub: classic, OAuth, user-to-server, installation and refresh tokens, and fine-grained PATs. + t('GitHub token', 'a GitHub classic token', `ghp_${run(36)}`), + t('GitHub token', 'a GitHub OAuth token', `gho_${run(36)}`), + t('GitHub token', 'a GitHub user-to-server token', `ghu_${run(36)}`), + t('GitHub token', 'a GitHub App installation token', `ghs_${run(36)}`), + t('GitHub token', 'a GitHub refresh token', `ghr_${run(36)}`), + t('GitHub token', 'a GitHub fine-grained token', `github_pat_11${run(20)}_${run(59)}`), + // Anthropic: API, setup-token (OAuth) and admin keys. + t('API key (sk-)', 'an Anthropic API key', `sk-ant-api03-${run(93)}AA`), + t('API key (sk-)', 'an Anthropic setup token', `sk-ant-oat01-${run(80)}AA`), + t('API key (sk-)', 'an Anthropic admin key', `sk-ant-admin01-${run(80)}AA`), + // Discord: a bot token (three base64url parts), older and newer first-part lengths. + t('Discord bot token', 'a Discord bot token', DISCORD), + t('Discord bot token', 'a newer Discord bot token', DISCORD_NEW), + // Slack: bot, user, legacy, refresh and rotation tokens, the app-level token and an incoming webhook. + t('Slack token', 'a Slack bot token', `xoxb-${j('-', '123456789012', '1234567890123', run(24))}`), + t('Slack token', 'a Slack user token', `xoxp-${j('-', '123456789012', '1234567890123', '1234567890123', hex(32))}`), + t('Slack token', 'a Slack workspace token', `xoxa-2-${run(40)}`), + t('Slack token', 'a Slack refresh token', `xoxr-${run(40)}`), + t('Slack token', 'a Slack legacy token', `xoxs-${run(40)}`), + t('Slack token', 'a Slack rotation token', `xoxe.xoxb-1-${run(140)}`), + t('Slack token', 'a Slack rotation refresh token', `xoxe-1-${run(140)}`), + t('Slack app-level token', 'a Slack app-level token', `xapp-${j('-', '1', 'A0123456789', '1234567890123', hex(64))}`), + t('Slack webhook', 'a Slack webhook', `https://hooks.slack.com/services/T0${upper(8)}/B0${upper(8)}/${run(24)}`, `${run(24)}`, { text: 'https://docs.example.test/services/team/guides/getting-started-page', part: 'getting-started-page' }), + // Linear: a personal API key and an OAuth token. + t('Linear key', 'a Linear API key', `lin_api_${run(40)}`), + t('Linear key', 'a Linear OAuth token', `lin_oauth_${run(64)}`), + // Tailscale: an auth key, an API access token and an OAuth client secret. + t('Tailscale key', 'a Tailscale auth key', `tskey-auth-k${upper(10)}CNTRL-${run(33)}`), + t('Tailscale key', 'a Tailscale API token', `tskey-api-k${upper(10)}CNTRL-${run(33)}`), + t('Tailscale key', 'a Tailscale client secret', `tskey-client-k${upper(10)}CNTRL-${run(33)}`), + // The rest of the shapes the shared redactor knows. + t('Stripe key', 'a Stripe live key', j('_', 'sk', 'live', run(24))), + t('Stripe key', 'a Stripe restricted key', j('_', 'rk', 'live', run(24))), + t('Google API key', 'a Google API key', `AI${'za'}${run(35, 'SyFAKE-fake_9')}`), + t('JWT', 'a JWT', j('.', `eyJ${'hbGciOiJIUzI1NiJ9'}`, `eyJ${'zdWIiOiJmYWtlLXVzZXIifQ'}`, run(43))), + t('AWS access key', 'an AWS access key id', `AKIA${upper(16)}`), + t('AWS access key', 'an AWS temporary key id', `ASIA${upper(16)}`), + t('private key', 'a PEM private key', PEM, PEM_BODY), + t('bearer token', 'a bearer token', `Bearer ${run(40, 'zY8xW7vU6')}`, run(40, 'zY8xW7vU6')), + t('32-byte hex key', 'a 32-byte hex key', `0x${hex(64)}`), + // Structures around a value, not token shapes. + { kind: 'password in a URL', what: 'a password in a URL', value: `postgres://app:${URL_PASSWORD}@db.example.test:5432/app`, secret: URL_PASSWORD, token: false }, + { kind: 'secret under a JSON key', what: 'a value under a JSON key', value: JSON.stringify({ id: 3, client_secret: JSON_VALUE }), secret: JSON_VALUE, token: false }, +]; + +/** Whether any 12-character piece of `secret` is still in `out`: a pattern that stops early leaks the rest. */ +export function leaks(out: string, secret: string, piece = 12): boolean { + if (secret.length <= piece) return out.includes(secret); + for (let i = 0; i + piece <= secret.length; i++) if (out.includes(secret.slice(i, i + piece))) return true; + return false; +} From 2c0142e1f0f7c45c089b7abaa562b23f80d0d60e Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:24:46 -0500 Subject: [PATCH 2/9] changelog.d: note the one shared list of token shapes (#921) The release note for #889's redaction change, as a fragment under the guardrail heading. Also rewraps the one-module lint's header comment (no code change). The first commit's body counts 13 of 15 tests failing on origin/main; with the two tests added in the adversarial pass (a Discord token after an underscore, a private key above the kept log lines) it is 15 of 17, as the PR body says. Test (fails without): none new; changelog.test.ts and changelog-fragments*.test.ts pass with it. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 3 +++ test/redaction-one-module.test.ts | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changelog.d/921-one-shared-redactor.md diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md new file mode 100644 index 000000000..b53a080d6 --- /dev/null +++ b/changelog.d/921-one-shared-redactor.md @@ -0,0 +1,3 @@ +### Guardrail changes: what each one does + +- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token and a Tailscale key. Each was redacted only by one module of its own before (the watchdog's incidents, the Tailscale step lines), so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. The dashboard's service-log file is redacted as one text before it is split, so a private key over several lines is masked whole. A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A bare 32-character key with no prefix stays unmasked when it stands alone: a commit hash or an id looks the same. A test fails on any module that redacts a token shape with its own pattern. diff --git a/test/redaction-one-module.test.ts b/test/redaction-one-module.test.ts index 694604ef3..efc782fb7 100644 --- a/test/redaction-one-module.test.ts +++ b/test/redaction-one-module.test.ts @@ -3,10 +3,11 @@ // drives each), and a module that needs a shape adds it there. A second list drifts: a Discord bot token was redacted // only in src/watchdog/incidents.ts, and passed through redactString everywhere else. // -// The lint: every `replace`, `replaceAll` or `split` call (by `.`, `?.` or `['...']`) in src/, web/src/ and scripts/ (TypeScript and JavaScript) -// whose pattern is a regex literal, a `new RegExp('...')` with a literal source, or a name bound to one (in the file, -// or else any file's const of that name), is run on each token stand-in (test/token-standins.ts). A match that -// takes in a whole token, and nothing of the words around it, is an own redactor for that shape. +// The lint: every `replace`, `replaceAll` or `split` call (by `.`, `?.` or `['...']`) in src/, web/src/ and scripts/ +// (TypeScript and JavaScript) whose pattern is a regex literal, a `new RegExp('...')` with a literal source, or a name +// bound to one (in the file, or else any file's const of that name), is run on each token stand-in +// (test/token-standins.ts). A match that takes in a whole token, and nothing of the words around it, is an own +// redactor for that shape. // Not caught: a pattern built at run time (a template with a substitution, String.raw, a concatenation, a function's // result), a regex passed through a parameter or kept in an object's property, a match found and then cut out by // string (match, then split and join), string methods other than these three, shell and other languages From 8633356dbfd62ebd1cdc2bcc9f8592b71b602d7f Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:59:47 -0500 Subject: [PATCH 3/9] redaction: prefixed dashboard keys, no own redactors, the journal redacted as one text (#889) Round 2, on the owner's two decisions and the final check (#921): - Dashboard keys are now wld_ and 32 random characters, in PATTERNS, so a key is redacted wherever it stands; a 32-character hex hash, a sha or a UUID is not. An older key still opens the dashboard and is not rewritten; doctor warns on each one it can read and says how to rotate it. - KNOWN_OWN_REDACTORS is gone. gh-login, the bench and the root updater redact through the shared redactor; what they masked on their own (a token cut short, a clone URL's user info, a value after a secret's name, a short bearer token) is BROAD in redact.ts, applied with broad: true and each module's own mask word. The updater stays one standalone file with a generated copy of redact.ts (scripts/gen-redact-vendor.mjs), held to the module byte for byte by test/redact-vendored.test.ts. - The dashboard's journal view redacts its entries as one text (redactRecords, with a margin of 100 entries), and clip() redacts before it cuts at 2,000 characters. - Shapes glued to a name after `_` and a lowercase bearer are found. Test (fails without): test/redact-records-keys.test.ts and test/redact-vendored.test.ts, 12 of 14 failing at 2c0142e1; with redact-shapes and redaction-one-module, 27 of 30 failing on origin/main 41f6dd42. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 9 +- scripts/bench/cli.ts | 6 +- scripts/gen-redact-vendor.mjs | 24 +++ scripts/lib/redact-vendor.mjs | 33 ++++ scripts/machine/worklane-update.cjs | 227 ++++++++++++++++++++++- src/dashboard.ts | 39 ++-- src/doctor.ts | 28 +++ src/events/redact.ts | 119 +++++++++++-- src/gh-login.ts | 5 +- src/merge-policy-weakening.ts | 8 +- test/dashboard-logs.test.ts | 3 +- test/redact-records-keys.test.ts | 238 +++++++++++++++++++++++++ test/redact-shapes.test.ts | 14 +- test/redact-vendored.test.ts | 42 +++++ test/redaction-one-module.test.ts | 16 +- test/token-standins.ts | 3 + 16 files changed, 753 insertions(+), 61 deletions(-) create mode 100644 scripts/gen-redact-vendor.mjs create mode 100644 scripts/lib/redact-vendor.mjs create mode 100644 test/redact-records-keys.test.ts create mode 100644 test/redact-vendored.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index b53a080d6..287a3bc7a 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -1,3 +1,10 @@ +### Upgrading (from v0.5.2) + +- **Dashboard keys now start with `wld_` (#889).** A new key is `wld_` and 32 random characters, a shape the redactor finds wherever it appears. A key from before keeps working until you rotate it. `doctor` warns on each older key it can read and says how to rotate it: remove the key file and restart what serves that dashboard. On a server with the hub, then run `scripts/setup/dashboards.sh` as root so the hub can read the new key. + ### Guardrail changes: what each one does -- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token and a Tailscale key. Each was redacted only by one module of its own before (the watchdog's incidents, the Tailscale step lines), so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. The dashboard's service-log file is redacted as one text before it is split, so a private key over several lines is masked whole. A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A bare 32-character key with no prefix stays unmasked when it stands alone: a commit hash or an id looks the same. A test fails on any module that redacts a token shape with its own pattern. +- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key and a dashboard key, and finds a token glued to a name after `_` (`MY_ghp_...`) and a lowercase `bearer`. Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. +- **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad rules (a token cut short, a clone URL's user info, a value after a secret's name, a short bearer token) keep everything those modules masked before. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. +- **The dashboard's service log** redacts its journal entries and log-file lines as one text, so a private key logged a line at a time is masked whole. It redacts a long line before cutting it to 2,000 characters. +- A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/bench/cli.ts b/scripts/bench/cli.ts index 2c11f8266..f59ed3112 100644 --- a/scripts/bench/cli.ts +++ b/scripts/bench/cli.ts @@ -30,6 +30,7 @@ import { realpathSync } from 'node:fs'; import { userInfo } from 'node:os'; import { fileURLToPath } from 'node:url'; +import { redactString } from '../../src/events/redact.js'; import { currentUser } from '../../src/os/index.js'; import { benchLane, runtimeDir } from './agent.js'; import { LOGIN_SWITCH_VAR, requireBenchUser, requireClaudePin, requireSubscription } from './auth.js'; @@ -74,8 +75,6 @@ export interface Check { /** A variable whose value may be a credential (container.ts SECRET_NAME without the daemon's names, which are paths). */ const SECRET_VAR = /KEY|TOKEN|SECRET|PASSW|CREDENTIAL|AUTH|COOKIE|SESSION/i; -/** Token shapes: Anthropic keys and OAuth tokens, GitHub tokens, Slack tokens, AWS key ids, JWTs. */ -const TOKEN_SHAPE = /sk-ant-[\w-]+|\bgh[opsur]_\w{16,}|\bgithub_pat_\w{16,}|\bxox[abprs]-[\w-]+|\bAKIA[0-9A-Z]{16}\b|\beyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]+/g; /** An email address: its domain ends in a letters-only label, so a package spec such as `name@0.0.79` is not one. */ const EMAIL = /[\w.+-]+@[\w-]+(?:\.[\w-]+)*\.[A-Za-z]{2,}\b/g; /** What scrub() keeps of one detail, and of one printed line. */ @@ -112,7 +111,8 @@ export function scrub(text: string, env: NodeJS.ProcessEnv, max = DETAIL_MAX): s for (const piece of [v.slice(0, n), v.slice(v.length - n)]) if (s.includes(piece)) s = s.split(piece).join('[redacted]'); } } - s = s.replace(TOKEN_SHAPE, '[redacted]').replace(EMAIL, '[email]').replace(/[\p{Cc}\p{Cf}]/gu, ' '); + // Token shapes through the shared redactor (#889), its broad rules too (a token cut short, a value after a secret's name). + s = redactString(s, { broad: true, mask: '[redacted]' }).replace(EMAIL, '[email]').replace(/[\p{Cc}\p{Cf}]/gu, ' '); return s.length > max ? `${s.slice(0, max - 3)}...` : s; } diff --git a/scripts/gen-redact-vendor.mjs b/scripts/gen-redact-vendor.mjs new file mode 100644 index 000000000..e8eacc35e --- /dev/null +++ b/scripts/gen-redact-vendor.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node +// Writes the root updater's generated copy of the shared redactor (scripts/lib/redact-vendor.mjs says why). +// Usage: node scripts/gen-redact-vendor.mjs [--write] (without --write: exit 1 when the copy is out of date) +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { blockIn, vendoredBlock, withBlock } from './lib/redact-vendor.mjs'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const machine = join(root, 'scripts', 'machine'); +const updaters = readdirSync(machine).filter((f) => f.endsWith('-update.cjs')); +if (updaters.length !== 1) throw new Error(`expected one scripts/machine/*-update.cjs, found ${updaters.length}`); +const file = join(machine, updaters[0]); +const text = readFileSync(file, 'utf8'); +const block = vendoredBlock(readFileSync(join(root, 'src', 'events', 'redact.ts'), 'utf8')); +if (blockIn(text) === block) { + console.log(`${updaters[0]}: the redactor copy is current`); +} else if (process.argv.includes('--write')) { + writeFileSync(file, withBlock(text, block)); + console.log(`${updaters[0]}: the redactor copy written`); +} else { + console.error(`${updaters[0]}: the redactor copy is out of date: run node scripts/gen-redact-vendor.mjs --write`); + process.exit(1); +} diff --git a/scripts/lib/redact-vendor.mjs b/scripts/lib/redact-vendor.mjs new file mode 100644 index 000000000..c12bc95ae --- /dev/null +++ b/scripts/lib/redact-vendor.mjs @@ -0,0 +1,33 @@ +// The root updater (scripts/machine/-update.cjs) is one standalone CommonJS file: it is installed alone, as root, +// and loads no engine module. So it carries the shared redactor (src/events/redact.ts) as a generated copy, between +// the two marker lines below: that module transpiled to CommonJS, unchanged. scripts/gen-redact-vendor.mjs writes it; +// test/redact-vendored.test.ts fails when the copy is not exactly what this makes of the module (#889). +import ts from 'typescript'; + +export const BEGIN = '// BEGIN generated from src/events/redact.ts: do not edit (node scripts/gen-redact-vendor.mjs --write)'; +export const END = '// END generated from src/events/redact.ts'; + +/** The block for `source` (src/events/redact.ts's text): the module, transpiled, as the object `sharedRedact`. */ +export function vendoredBlock(source) { + const out = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, removeComments: true }, + fileName: 'redact.ts', + }); + // Lines are kept as transpiled (no indent added): a template literal over two lines must not change. + return [BEGIN, 'const sharedRedact = (() => {', 'const exports = {};', out.outputText.trimEnd(), 'return exports;', '})();', END].join('\n'); +} + +/** The block in `text` (the updater's source), markers included; null when its markers are missing or out of order. */ +export function blockIn(text) { + const a = text.indexOf(BEGIN); + const b = text.indexOf(END); + if (a === -1 || b === -1 || b < a || text.indexOf(BEGIN, a + 1) !== -1) return null; + return text.slice(a, b + END.length); +} + +/** `text` with its block replaced by `block`. Throws when the markers are missing. */ +export function withBlock(text, block) { + const old = blockIn(text); + if (old === null) throw new Error('the updater has no generated redactor block (its BEGIN and END marker lines)'); + return text.replace(old, () => block); +} diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 489b2fbe2..105fa7edf 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -108,6 +108,220 @@ const fs = require('node:fs'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); +// The shared redactor (src/events/redact.ts), as a generated copy: this file is installed alone and loads no engine +// module (scripts/lib/redact-vendor.mjs; test/redact-vendored.test.ts checks the copy is exactly that module). +// BEGIN generated from src/events/redact.ts: do not edit (node scripts/gen-redact-vendor.mjs --write) +const sharedRedact = (() => { +const exports = {}; +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; +exports.secretKinds = secretKinds; +exports.redactString = redactString; +exports.redactRecords = redactRecords; +exports.redact = redact; +const credentialShaped = (v) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); +const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'jwt', 'bearer', 'credential', 'credentials', 'authorization']); +const CREDENTIAL_ENDINGS = ['token', 'secret', 'password', 'passwd', 'accesskey']; +const credentialWord = (w) => CREDENTIAL_WORDS.has(w) || CREDENTIAL_ENDINGS.some((e) => w.endsWith(e)); +const KEY_QUALIFIERS = new Set(['api', 'access', 'secret', 'private', 'signing', 'encryption', 'master', 'client', 'app', 'auth', 'account', 'service', 'license', 'deploy', 'admin', 'webhook', 'session', 'ssh', 'hmac', 'consumer']); +function credentialKey(k) { + const words = k + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Z])([A-Z][a-z])/g, '$1 $2') + .toLowerCase() + .split(/[ _-]+/) + .filter(Boolean); + const last = words.at(-1) ?? ''; + if (words.some(credentialWord) || last === 'auth') + return true; + if (last === 'key') + return words.length === 1 || KEY_QUALIFIERS.has(words.at(-2)); + return last.endsWith('key') && KEY_QUALIFIERS.has(last.slice(0, -3)); +} +exports.DASHBOARD_KEY_PREFIX = 'wld_'; +const DASHBOARD_KEY_BODY = '[A-Za-z0-9_-]{32,}'; +const isDashboardKey = (key) => new RegExp(`^${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}$`).test(key); +exports.isDashboardKey = isDashboardKey; +const PATTERNS = [ + [new RegExp(`(^|[^A-Za-z0-9_-])${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `$1${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], + [/\b(gh[pousr]_[A-Za-z0-9]{30,})\b/g, 'gh*_[REDACTED]', 'GitHub token'], + [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], + [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], + [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], + [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]', 'Google API key'], + [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*/g, '$1[REDACTED JWT]', 'JWT'], + [/\b(AKIA|ASIA)[A-Z0-9]{16}\b/g, 'AKIA[REDACTED]', 'AWS access key'], + [/\bxoxe[.-][A-Za-z0-9.-]{10,}\b/g, 'xoxe-[REDACTED]', 'Slack token'], + [/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g, 'xox-[REDACTED]', 'Slack token'], + [/\bxapp-[A-Za-z0-9-]{10,}\b/g, 'xapp-[REDACTED]', 'Slack app-level token'], + [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}\b/g, 'lin_[REDACTED]', 'Linear key'], + [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], + [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], + [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]', 'private key'], + [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], + [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], + [/\b0x[0-9a-fA-F]{64}\b/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], + [ + /(\\?")([A-Za-z0-9_-]+)\1(\s*:\s*)\1([A-Za-z0-9_\-.+/=~]{16,})\1/g, + (m, q, k, sep, v) => (credentialKey(k) && credentialShaped(v) ? `${q}${k}${q}${sep}${q}[REDACTED]${q}` : m), + 'secret under a JSON key', + ], + [/(^|[^A-Za-z0-9])[A-Za-z0-9_-]{23,28}\.[A-Za-z0-9_-]{6,7}\.[A-Za-z0-9_-]{27,}\b/g, '$1[REDACTED]', 'Discord bot token'], +]; +const BROAD = [ + [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], + [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], + [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], + [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], + [/(^|[^a-z0-9+.-])([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, (k) => `$1$2${k}@`, 'user info in a URL'], + [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], +]; +const CODE_HEADS = new Set(['this', 'self', 'super', 'opts', 'options', 'o', 'ctx', 'context', 'config', 'cfg', 'conf', 'settings', 'props', 'state', 'env', 'process', 'module', 'exports', 'window', 'document', 'globalThis', 'args', 'params', 'req', 'res', 'request', 'response', 'obj', 'result', 'item', 'e', 'err', 'x']); +function internalHost(m, pre, name) { + const lower = name.toLowerCase(); + const labels = name.split('.').slice(0, lower.endsWith('.home.arpa') || lower.endsWith('.ts.net') ? -2 : -1); + const suffix = name.slice(labels.join('.').length + 1); + const prose = suffix !== suffix.toLowerCase() && suffix !== suffix.toUpperCase(); + const code = !labels.some((l) => l.includes('-')) && (CODE_HEADS.has(labels[0]) || labels.some((l) => /[a-z][A-Z]/.test(l))); + return prose || code ? m : `${pre}[internal-host]`; +} +const OCTET = '(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)'; +const PRIVATE_V4 = [ + `10\\.${OCTET}`, + '172\\.(?:1[6-9]|2\\d|3[01])', + '192\\.168', + '100\\.(?:6[4-9]|[7-9]\\d|1[01]\\d|12[0-7])', + '169\\.254', +].join('|'); +const IDENTIFIERS = [ + [/(^|[^A-Za-z0-9._%+-])(?!git@)[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}(?![A-Za-z0-9-])/g, '$1[email]', 'email'], + [ + /(^|[^\w.~/:-]|file:\/\/)\/(?:home|Users|mnt\/[A-Za-z]\/Users)\/([^\s/"'`<>()[\]{},;|\\]+)/g, + (m, pre, user) => (user === 'runner' ? m : `${pre}~`), + 'home path', + ], + [/(^|[^\w])[A-Za-z]:[\\/][Uu]sers[\\/]([^\\/\s"'`<>|]+)/g, '$1~', 'home path'], + [/(^|[^A-Za-z0-9./-]|\/\/)([A-Za-z0-9][A-Za-z0-9.-]*\.(?:internal|localdomain|lan|corp|intranet|home\.arpa|ts\.net|local))(?![A-Za-z0-9-]|\.[A-Za-z0-9])/gi, internalHost, 'internal host'], + [new RegExp(`(^|[^\\w.])(?:${PRIVATE_V4})\\.${OCTET}\\.${OCTET}(?![\\w-]|\\.\\d)`, 'g'), '$1[private-ip]', 'private IP'], + [/(^|[^\w:])f(?:[cd][0-9a-f]{2}|e[89ab][0-9a-f]):(?:[0-9a-f]{0,4}:){1,6}[0-9a-f]{0,4}(?![\w:])/gi, '$1[private-ip]', 'private IP'], +]; +exports.REDACTION_PATTERNS = [ + ...PATTERNS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'secret' })), + ...BROAD.map(([re, rep, kind]) => ({ kind, re, rep: rep('[REDACTED]'), group: 'broad' })), + ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'identifier' })), +]; +const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', exports.DASHBOARD_KEY_PREFIX]; +function afterUnderscore(s, f) { + const cuts = []; + for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) + if (GLUED_PREFIXES.some((p) => s.startsWith(p, at + 1))) + cuts.push(at + 1); + if (!cuts.length) + return s; + let out = ''; + let from = 0; + for (const c of [...cuts, s.length]) { + out += from === 0 ? s.slice(0, c) : f(s.slice(from, c)); + from = c; + } + return out; +} +const expand = (rep, m, groups) => rep.replace(/\$(\d|\$|&)/g, (_, k) => (k === '$' ? '$' : k === '&' ? m : String(groups[Number(k) - 1] ?? ''))); +const breaks = (s) => s.split('\n').length - 1; +function applyKeepingLines(s, re, rep) { + return s.replace(re, (m, ...rest) => { + const groups = rest.slice(0, -2).filter((g) => typeof g !== 'object'); + const out = typeof rep === 'string' ? expand(rep, m, groups) : rep(m, ...groups); + return out + '\n'.repeat(Math.max(0, breaks(m) - breaks(out))); + }); +} +exports.TRACKER = Object.freeze({ identifiers: true }); +const apply = (s, re, rep) => (typeof rep === 'string' ? s.replace(re, rep) : s.replace(re, rep)); +const MACHINE_BLOCKS = [ + ['```done_when', /[^\S\n]*\n/y], + ['```research', /[ \t]*\r?\n/y], +]; +function outsideMachineBlocks(s, f) { + const spans = []; + for (const [open, lineBreak] of MACHINE_BLOCKS) { + for (let at = s.indexOf(open); at !== -1;) { + lineBreak.lastIndex = at + open.length; + if (!lineBreak.test(s)) { + at = s.indexOf(open, at + open.length); + continue; + } + const close = s.indexOf('\n```', lineBreak.lastIndex); + if (close === -1) + break; + spans.push([at, close + 4]); + at = s.indexOf(open, close + 4); + } + } + if (!spans.length) + return f(s); + spans.sort((a, b) => a[0] - b[0]); + let out = ''; + let from = 0; + for (const [a, b] of spans) { + if (a < from) + continue; + out += f(s.slice(from, a)) + s.slice(a, b); + from = b; + } + return out + f(s.slice(from)); +} +function secretKinds(s) { + const kinds = new Set(); + for (const [re, rep, kind] of PATTERNS) { + re.lastIndex = 0; + if (apply(s, re, rep) !== s) + kinds.add(kind); + } + return [...kinds].sort(); +} +function redactString(s, o = {}) { + const put = o.keepLines ? applyKeepingLines : apply; + let out = s; + if (o.broad) { + for (const [re, rep, kind] of PATTERNS) + if (kind === 'private key') + out = put(out, re, rep); + const mask = (o.mask ?? '[REDACTED]').replace(/\$/g, '$$$$'); + for (const [re, rep] of BROAD) + out = put(out, re, rep(mask)); + } + const secrets = (t) => PATTERNS.reduce((x, [re, rep]) => put(x, re, rep), t); + out = afterUnderscore(secrets(out), secrets); + if (o.identifiers) + out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); + return out; +} +function redactRecords(records, o = {}) { + const lines = redactString(records.join('\n'), { ...o, keepLines: true }).split('\n'); + const out = []; + let at = 0; + for (const r of records) { + const n = breaks(r) + 1; + out.push(lines.slice(at, at + n).join('\n')); + at += n; + } + return out; +} +function redact(value, o = {}) { + if (typeof value === 'string') + return redactString(value, o); + if (Array.isArray(value)) + return value.map((v) => redact(v, o)); + if (value && typeof value === 'object') + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, redact(v, o)])); + return value; +} +return exports; +})(); +// END generated from src/events/redact.ts + // Fixed paths: the installed updater takes none from its caller. const CONFIG = '/etc/worklane/updates.json'; const LOG = '/var/lib/worklane/updates.jsonl'; @@ -894,14 +1108,13 @@ function health(samples, startedAt, now) { return now - startedAt >= HEALTH_MS ? 'failed' : 'wait'; } -/** Secrets that could appear in a service's journal, replaced before anything is logged. */ +/** + * Secrets that could appear in a service's journal, replaced before anything is logged: the shared redactor's every + * shape (sharedRedact, the generated copy of src/events/redact.ts at the top of this file, #889), and its broad rules + * (a token cut short, a clone URL's user:token@, a value after a secret's name, a short bearer token), as [redacted]. + */ function redact(text) { - return String(text) - .replace(/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[redacted key]') - .replace(/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, '[redacted]') - .replace(/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 [redacted]') - .replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, '$1[redacted]@') // WHY: a clone URL can carry user:token@ - .replace(/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, '$1$2[redacted]'); + return sharedRedact.redactString(String(text), { broad: true, mask: '[redacted]' }); } /** diff --git a/src/dashboard.ts b/src/dashboard.ts index def8d9bfb..1c5e8cf43 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -36,7 +36,7 @@ import { EventCache, type OpenRead } from './events/cache.js'; import { compress, compressibleType, MIN_COMPRESS_BYTES, mayCompressApi, requestEncoding, StaticCompressed, type Encoding } from './dashboard-compress.js'; import { dashboardProof, NONCE, PROOF_PATH } from './dashboard-proof.js'; import { localDay } from './local-day.js'; -import { redactString } from './events/redact.js'; +import { DASHBOARD_KEY_PREFIX, redactRecords, redactString } from './events/redact.js'; import type { EventPayload, StoredEvent } from './events/types.js'; import { acceptGit, acceptMerge, protectionView, type AcceptGit, type AcceptTarget } from './main-watch.js'; import { deployGateView } from './deploy-gate/gate.js'; @@ -884,8 +884,15 @@ const journalctl: JournalReader = (unit, lines) => { return { status: r.status, stdout: r.stdout ?? '', stderr: `${r.stderr ?? ''}${r.error ? r.error.message : ''}` }; }; -const clip = (s: string) => redactString(s.length > 2000 ? `${s.slice(0, 2000)} …` : s); -/** Lines read above a service log file's kept lines, so a private key that starts there is redacted whole (a 4096-bit key is about 50 lines). */ +/** A message cut to 2,000 characters, redacted first: a cut made before would leave a long secret's body unfound (#889). */ +const clip = (s: string) => { + const r = redactString(s); + return r.length > 2000 ? `${r.slice(0, 2000)} …` : r; +}; +/** + * Records read above the kept ones (journal entries or log-file lines), so a private key that starts there is + * redacted whole (a 4096-bit key is about 50 lines). + */ const KEY_MARGIN = 100; /** @@ -898,20 +905,25 @@ const KEY_MARGIN = 100; export function readServiceLog(src: LogSource, lines = 200, read: JournalReader = journalctl): ServiceLog { let problem: string | null = null; if (src.unit) { - const r = read(src.unit, lines); + // KEY_MARGIN entries more than are shown, read for redaction only. + const r = read(src.unit, lines + KEY_MARGIN); if (r.status === 0) { - const entries: LogEntry[] = []; + const records: LogEntry[] = []; for (const l of r.stdout.split('\n')) { if (!l.trim()) continue; try { const j = JSON.parse(l) as { MESSAGE?: string | number[] | null; PRIORITY?: string; __REALTIME_TIMESTAMP?: string }; const msg = typeof j.MESSAGE === 'string' ? j.MESSAGE : Array.isArray(j.MESSAGE) ? Buffer.from(j.MESSAGE).toString('utf8') : ''; const us = Number(j.__REALTIME_TIMESTAMP); - entries.push({ at: Number.isFinite(us) && us > 0 ? new Date(us / 1000).toISOString() : null, priority: j.PRIORITY !== undefined && /^\d$/.test(j.PRIORITY) ? Number(j.PRIORITY) : null, message: clip(msg) }); + records.push({ at: Number.isFinite(us) && us > 0 ? new Date(us / 1000).toISOString() : null, priority: j.PRIORITY !== undefined && /^\d$/.test(j.PRIORITY) ? Number(j.PRIORITY) : null, message: msg }); } catch { // not a journal record } } + // journald keeps a service's output one line per entry: the entries are redacted as one text (#889), so a private + // key logged a line at a time is found whole. An entry that was inside one comes back empty and is left out. + const shown = redactRecords(records.map((e) => e.message)); + const entries = records.flatMap((e, i) => (shown[i]!.trim() || !e.message.trim() ? [{ ...e, message: clip(shown[i]!) }] : [])).slice(-lines); if (entries.length) return { source: 'journal', unit: src.unit, entries, problem: null }; problem = `no lines of ${src.unit} this user can read. Either the service hasn't logged yet, or the journal isn't kept on disk split per user, so the service's own user can't read its lines: run scripts/setup/journal.sh on the machine (as an admin).`; } else { @@ -920,11 +932,11 @@ export function readServiceLog(src: LogSource, lines = 200, read: JournalReader } if (src.file && existsSync(src.file)) { const text = readFileSync(src.file, 'utf8'); - // The last lines, and KEY_MARGIN more above them, are redacted as one text before they are split, so a secret over - // several lines (a private key, even one that starts above the kept lines) is found whole (#889); each line is - // redacted again as it is clipped. + // The last lines, and KEY_MARGIN more above them, are redacted as one text, so a secret over several lines (a + // private key, even one that starts above the kept lines) is found whole (#889); each line is redacted again as it + // is clipped. const tail = text.split('\n').filter((l) => l.trim()).slice(-(lines + KEY_MARGIN)); - const kept = redactString(tail.join('\n')).split('\n').filter((l) => l.trim()).slice(-lines); + const kept = redactRecords(tail).filter((l) => l.trim()).slice(-lines); const entries = kept.map((message) => ({ at: null, priority: null, message: clip(message) })); return { source: 'file', unit: src.unit, entries, problem: entries.length ? null : problem }; } @@ -1047,11 +1059,16 @@ export function loopbackHostHeader(host: string | undefined, port: number | unde /** GET: the key page's help (the key file's path and a copy command), loopback only, never the key. */ export const KEY_HELP_PATH = '/api/key-help'; +/** + * The dashboard's key: the one in its file, whatever its shape (a key from before #889 keeps working until it is + * rotated; doctor asks for that), or a new one, DASHBOARD_KEY_PREFIX and 24 random bytes, a shape the shared + * redactor always finds. + */ export function dashboardToken(stateDir: string): string { const f = join(stateDir, 'dashboard-token'); if (existsSync(f)) return readFileSync(f, 'utf8').trim(); mkdirSync(stateDir, { recursive: true }); - const t = randomBytes(24).toString('base64url'); + const t = `${DASHBOARD_KEY_PREFIX}${randomBytes(24).toString('base64url')}`; writeFileSync(f, t, { mode: 0o600 }); return t; } diff --git a/src/doctor.ts b/src/doctor.ts index 85a39b680..7d4c20bfc 100644 --- a/src/doctor.ts +++ b/src/doctor.ts @@ -33,6 +33,7 @@ import { checkSetup, groupLevel, GROUPS, stepLevel, systemProbe, type SetupConte import { modelWarnings } from './model-routing.js'; import { HUB_UNIT_AS_ROOT, OWNER_TERMINAL_RERUN, ownerTerminalNotice, ownerTerminalRerun, type TerminalPaths } from './console/owner-terminal-removed.js'; import { voiceSiriLeftoverCheck } from './doctor-voice-removed.js'; +import { DASHBOARD_KEY_PREFIX, isDashboardKey } from './events/redact.js'; /** off: not available in this setup's mode (said why; never a failure). */ export type Level = 'ok' | 'warn' | 'fail' | 'off'; @@ -257,9 +258,36 @@ export function doctor(rootArg: string, opts: { agentshield?: boolean; /** A pol if (cfg?.tests.test_run) out.push(networkCheck()); if (opts.agentshield) out.push(agentShield(root)); out.push(...policyRootChecks()); + // #889: each dashboard key this user can read, the local one and each instance's: an older key asks for rotation. + for (const [where, dir] of [['local', projectStateDir(root)], ...instances.map((n) => [n, join(instancesDir(), n, 'state')])] as [string, string][]) { + const k = dashboardKeyCheck(join(dir, 'dashboard-token'), where); + if (k) out.push(k); + } return out; } +/** + * A dashboard key file (#889): a key from before keys had DASHBOARD_KEY_PREFIX still opens the dashboard, but a shape + * the redactor can't tell from a commit hash or an id, so it may show in a log or a post in clear: rotate it. Null + * when there is no file or this user can't read it. The key is read and never shown. + */ +export function dashboardKeyCheck(file: string, where: string, read: (f: string) => string = (f) => readFileSync(f, 'utf8')): DoctorCheck | null { + let key: string; + try { + key = read(file).trim(); + } catch { + return null; + } + if (!key) return null; + const name = `dashboard key ${where}`; + if (isDashboardKey(key)) return { name, level: 'ok', detail: `${file}: the current key format (${DASHBOARD_KEY_PREFIX}...), redacted wherever it appears` }; + return { + name, + level: 'warn', + detail: `${file} holds a dashboard key in the older format (no ${DASHBOARD_KEY_PREFIX} prefix). It still works, but the redactor can't tell it from a commit hash or an id, so it could show in a log or a post. Rotate it: remove the file and restart what serves that dashboard (the instance's service, or \`${BRAND.cli} dashboard\`); a new key is written, and the page asks for it once. On a server with the hub, then run scripts/setup/dashboards.sh as root so the hub can read the new key.`, + }; +} + /** * A failure when the instance's newest sandbox check (its coordinator's `testrun.sandbox` event) said this machine * can't give a test run a user namespace of its own (util-linux older than 2.38, or no setpriv), with the fix; else diff --git a/src/events/redact.ts b/src/events/redact.ts index 0212f4c06..a816afc4d 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -44,8 +44,21 @@ function credentialKey(k: string): boolean { return last.endsWith('key') && KEY_QUALIFIERS.has(last.slice(0, -3)); } +/** + * What every dashboard key starts with (#889, owner's decision): src/dashboard.ts dashboardToken writes `wld_` and 32 + * base64url characters, a shape no commit hash, digest or engine id has, so a key is redacted wherever it appears. + * Fixed, not taken from BRAND: renaming the product must not change the shape of keys already issued. A key from + * before (32 characters, no prefix) still opens the dashboard; doctor asks for it to be rotated. + */ +export const DASHBOARD_KEY_PREFIX = 'wld_'; +const DASHBOARD_KEY_BODY = '[A-Za-z0-9_-]{32,}'; +/** Whether `key` is a dashboard key of the current shape (prefixed). */ +export const isDashboardKey = (key: string): boolean => new RegExp(`^${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}$`).test(key); + // Each pattern with what kind of secret it finds (secretKinds names the kind, never the value). const PATTERNS: [RegExp, Replacement, string][] = [ + // A dashboard key: starts only where a run of key characters starts (`wld_wld_...` is one run), so linear. + [new RegExp(`(^|[^A-Za-z0-9_-])${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `$1${DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], [/\b(gh[pousr]_[A-Za-z0-9]{30,})\b/g, 'gh*_[REDACTED]', 'GitHub token'], [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], @@ -69,7 +82,8 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // prefix and the rest of its run, however short (src/tailscale-setup.ts redacted them this way on its own). [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]', 'private key'], - [/\b(Bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], + // The scheme is case-insensitive (RFC 9110): `bearer` and `BEARER` too (#889). + [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], // Credentials inside connection strings: keep scheme, user and host. [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], // 0x-prefixed 32-byte hex (EVM private keys and the like). @@ -90,9 +104,26 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // is bounded (23 to 28 characters, then a dot), so each start reads a bounded stretch: linear (redact-shapes.test.ts). [/(^|[^A-Za-z0-9])[A-Za-z0-9_-]{23,28}\.[A-Za-z0-9_-]{6,7}\.[A-Za-z0-9_-]{27,}\b/g, '$1[REDACTED]', 'Discord bot token'], ]; -// Not covered, by decision (#889): a bare 32-character key with no prefix, such as a dashboard key standing alone. A -// commit hash, a digest and the engine's own ids (randomBytes(16) in hex) have the same shape. A key in a Bearer -// header, a JSON value under a credential key, or a URL's password is covered; a key is never put in a URL. +// Not covered: a bare 32-character string with no prefix (a dashboard key from before #889 among them): a commit +// hash, a digest and the engine's own ids have the same shape. New dashboard keys carry DASHBOARD_KEY_PREFIX. + +/** + * The broad rules (`broad: true`), for text whose source may cut a token short or name a secret by its key: gh's + * output (src/gh-login.ts), the bench's printed lines (scripts/bench/cli.ts) and the root updater's files + * (scripts/machine/-update.cjs, which carries a generated copy of this module). Applied before PATTERNS, each + * puts `mask` (default [REDACTED]) in place of what it finds. They were those modules' own rules (#889): a known + * token prefix and a short run, a URL's user info, a value after a secret's name, a short bearer token, and a JWT + * after a hyphen. Each starts only where a run starts, or reads a bounded stretch from each start: linear. + */ +const BROAD: [RegExp, (mask: string) => string, string][] = [ + [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], + [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], + [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], + [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], + [/(^|[^a-z0-9+.-])([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, (k) => `$1$2${k}@`, 'user info in a URL'], + [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], +]; /** * The first word of a dotted name that is code, not a host: `this.internal`, `opts.corp`, `config.lan.x`. Only @@ -157,16 +188,58 @@ const IDENTIFIERS: [RegExp, Replacement, string][] = [ /** * Every pattern with its kind, for the linear-time test (test/redact.test.ts) and the shape tests - * (test/redact-shapes.test.ts): `secret` for a secret's pattern (applied by default), not for an identifier's. + * (test/redact-shapes.test.ts): `secret` for a secret's pattern (applied by default), `broad` for one applied with + * `broad: true` (shown with the default mask), `identifier` for one applied with `identifiers: true`. */ -export const REDACTION_PATTERNS: readonly { kind: string; re: RegExp; rep: Replacement; secret: boolean }[] = [ - ...PATTERNS.map(([re, rep, kind]) => ({ kind, re, rep, secret: true })), - ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, secret: false })), +export const REDACTION_PATTERNS: readonly { kind: string; re: RegExp; rep: Replacement; group: 'secret' | 'broad' | 'identifier' }[] = [ + ...PATTERNS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'secret' as const })), + ...BROAD.map(([re, rep, kind]) => ({ kind, re, rep: rep('[REDACTED]'), group: 'broad' as const })), + ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'identifier' as const })), ]; export interface RedactTextOptions { /** Also redact emails, internal hostnames, private IPs and home paths' user names (text for an issue, comment or PR). */ identifiers?: boolean; + /** Also apply the broad rules (BROAD), first: a token prefix cut short, a URL's user info, a value after a secret's name. */ + broad?: boolean; + /** What a broad rule puts in place of what it finds; [REDACTED] when unset. */ + mask?: string; + /** Keep the text's line count: a match over several lines (a private key) leaves its line breaks (redactRecords). */ + keepLines?: boolean; +} + +/** + * The prefixes of the shapes whose patterns start at `\b` (or after a character that isn't `_`): after an underscore + * (`MY_ghp_...`, `x_sk-...`) a second pass reads from the prefix on (afterUnderscore), so a key glued to a name is + * found too (#889). The first pass is unchanged, so nothing it found is lost. + */ +const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', DASHBOARD_KEY_PREFIX]; + +/** `s` with `f` applied again from each underscore that a known prefix follows: indexOf scans, so linear. */ +function afterUnderscore(s: string, f: (part: string) => string): string { + const cuts: number[] = []; + for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) if (GLUED_PREFIXES.some((p) => s.startsWith(p, at + 1))) cuts.push(at + 1); + if (!cuts.length) return s; + let out = ''; + let from = 0; + for (const c of [...cuts, s.length]) { + out += from === 0 ? s.slice(0, c) : f(s.slice(from, c)); + from = c; + } + return out; +} + +/** `$1`, `$&` and `$$` in a replacement string, as String.prototype.replace reads them. */ +const expand = (rep: string, m: string, groups: unknown[]) => rep.replace(/\$(\d|\$|&)/g, (_, k: string) => (k === '$' ? '$' : k === '&' ? m : String(groups[Number(k) - 1] ?? ''))); +const breaks = (s: string) => s.split('\n').length - 1; + +/** As apply, but each replacement keeps as many line breaks as what it replaced. */ +function applyKeepingLines(s: string, re: RegExp, rep: Replacement): string { + return s.replace(re, (m: string, ...rest: unknown[]) => { + const groups = rest.slice(0, -2).filter((g) => typeof g !== 'object'); + const out = typeof rep === 'string' ? expand(rep, m, groups) : rep(m, ...(groups as string[])); + return out + '\n'.repeat(Math.max(0, breaks(m) - breaks(out))); + }); } /** The options for text bound for an issue, a comment or a pull request (src/backlog): secrets and identifiers. */ @@ -225,9 +298,35 @@ export function secretKinds(s: string): string[] { } export function redactString(s: string, o: RedactTextOptions = {}): string { + const put = o.keepLines ? applyKeepingLines : apply; let out = s; - for (const [re, rep] of PATTERNS) out = apply(out, re, rep); - if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => apply(t, re, rep), part)); + if (o.broad) { + // A private key first: a broad rule (a value after `TOKEN=`) would take its BEGIN line and leave its body unfound. + for (const [re, rep, kind] of PATTERNS) if (kind === 'private key') out = put(out, re, rep); + // A `$` in the mask is taken literally. + const mask = (o.mask ?? '[REDACTED]').replace(/\$/g, '$$$$'); + for (const [re, rep] of BROAD) out = put(out, re, rep(mask)); + } + const secrets = (t: string) => PATTERNS.reduce((x, [re, rep]) => put(x, re, rep), t); + out = afterUnderscore(secrets(out), secrets); + if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); + return out; +} + +/** + * Records shown one after another (a service's journal entries, a log file's lines), redacted as one text so a secret + * over several of them (a private key logged a line at a time) is found whole; the same number of records comes back, + * each with as many lines as it had (a record inside a secret comes back empty or holding the marker). + */ +export function redactRecords(records: readonly string[], o: RedactTextOptions = {}): string[] { + const lines = redactString(records.join('\n'), { ...o, keepLines: true }).split('\n'); + const out: string[] = []; + let at = 0; + for (const r of records) { + const n = breaks(r) + 1; + out.push(lines.slice(at, at + n).join('\n')); + at += n; + } return out; } diff --git a/src/gh-login.ts b/src/gh-login.ts index 4fb87fbac..94274cdd2 100644 --- a/src/gh-login.ts +++ b/src/gh-login.ts @@ -11,6 +11,7 @@ import { spawn, spawnSync } from './os/child-process.js'; import { findTool, withToolPath } from './tool-path.js'; import { outboundEnv } from './network.js'; +import { redactString } from './events/redact.js'; export const GH_DEVICE_URL = 'https://github.com/login/device'; /** What runs: the web (device) flow for github.com, git over https, so gh asks nothing else. */ @@ -21,7 +22,6 @@ export const GH_LOGIN_TIMEOUT_MS = 15 * 60_000; export const GH_STATUS_EVERY_MS = 3_000; const CODE = /one-time code:\s*([A-Z0-9]{4}-[A-Z0-9]{4})\b/; -const TOKENISH = /\b(gh[opusr]_[A-Za-z0-9_]{8,}|github_pat_[A-Za-z0-9_]{8,})/g; /** The child gh process, as this module uses it (tests stand in for it). */ export interface LoginChild { @@ -85,7 +85,8 @@ export function safeLine(text: string): string { .map((l) => l.trim()) .filter((l) => l && !CODE.test(l) && !l.includes(GH_DEVICE_URL)) .at(-1) ?? ''; - return line.replace(TOKENISH, '[hidden]').slice(0, 200); + // The shared redactor's broad rules (a GitHub token from 8 characters: gh's output may cut one short), as [hidden]. + return redactString(line, { broad: true, mask: '[hidden]' }).slice(0, 200); } /** The relay: one login at a time, per dashboard. */ diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 54f7827e2..e961512e9 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -63,8 +63,7 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'scripts/lib/denylist.mjs', table: 'SKIP', kind: 'never-grow' }, { path: 'src/guardrails/engine.ts', table: 'AGENT_EDITABLE', kind: 'never-grow' }, { path: 'scripts/bench/pins.ts', table: 'SELECTION_RULES', kind: 'never-grow' }, - // #889: the modules that still redact a token shape on their own, and the one-module lint's good fixtures. - { path: 'test/redaction-one-module.test.ts', table: 'KNOWN_OWN_REDACTORS', kind: 'never-grow' }, + // #889: the one-module lint's good fixtures. { path: 'test/redaction-one-module.test.ts', table: 'GOOD_SOURCES', kind: 'never-grow' }, // The lints' rules and bad fixtures, and the guards' lists: taking one out is a loosening. { path: 'test/change-files-renames.test.ts', table: 'READER_WORDS', kind: 'never-shrink' }, @@ -89,6 +88,11 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'test/runner-golden.test.ts', table: 'GUARD', kind: 'never-shrink' }, // #889: the shared redactor's one list of credential shapes, a stand-in for each, and the one-module lint's bad fixtures. { path: 'src/events/redact.ts', table: 'PATTERNS', kind: 'never-shrink' }, + { path: 'src/events/redact.ts', table: 'BROAD', kind: 'never-shrink' }, + // The root updater's generated copy of that module (test/redact-vendored.test.ts holds it to the module). + { path: `scripts/machine/${BRAND.cli}-update.cjs`, table: 'PATTERNS', kind: 'never-shrink' }, + { path: `scripts/machine/${BRAND.cli}-update.cjs`, table: 'CREDENTIAL_WORDS', kind: 'never-shrink' }, + { path: 'test/redact-records-keys.test.ts', table: 'CUT_SHORT', kind: 'never-shrink' }, { path: 'test/token-standins.ts', table: 'SHAPES', kind: 'never-shrink' }, { path: 'test/redaction-one-module.test.ts', table: 'BAD_SOURCES', kind: 'never-shrink' }, // The guards' own tables (their files are guard files too; the table names what was taken out). diff --git a/test/dashboard-logs.test.ts b/test/dashboard-logs.test.ts index efbee2584..e07eeeacb 100644 --- a/test/dashboard-logs.test.ts +++ b/test/dashboard-logs.test.ts @@ -19,7 +19,8 @@ test('the service log comes from the unit\'s journal, read as this user, with se return { status: 0, stderr: '', stdout: [rec('tick: 0 dispatched', 1_700_000_000_000_000), rec(`push failed with ${token}`, 1_700_000_001_000_000, '3'), rec([104, 105], 1_700_000_002_000_000), 'not json'].join('\n') }; }; const log = readServiceLog({ unit: 'x.service', file: null }, 50, read); - assert.deepEqual(asked, [['x.service', 50]]); + // 50 shown, and 100 more above them read for redaction only (#889: a private key that starts above them). + assert.deepEqual(asked, [['x.service', 150]]); assert.equal(log.source, 'journal'); assert.equal(log.problem, null); assert.deepEqual(log.entries.map((e) => [e.priority, e.message]), [[6, 'tick: 0 dispatched'], [3, 'push failed with gh*_[REDACTED]'], [6, 'hi']]); diff --git a/test/redact-records-keys.test.ts b/test/redact-records-keys.test.ts new file mode 100644 index 000000000..9d30c01f9 --- /dev/null +++ b/test/redact-records-keys.test.ts @@ -0,0 +1,238 @@ +// #889 round 2 (owner's decisions and the final check): dashboard keys carry a prefix the shared redactor always +// finds; gh-login, the bench and the root updater redact through the shared redactor (its broad rules for what they +// masked on their own), with no exception list; the dashboard's journal view redacts its entries as one text and +// before it cuts; a shape glued to a name after `_`, and a lowercase bearer, are found. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as redactModule from '../src/events/redact.js'; +import { redactString, REDACTION_PATTERNS } from '../src/events/redact.js'; +import { EventLog } from '../src/events/log.js'; +import { FileBacklog } from '../src/backlog/file.js'; +import { buildDigest } from '../src/superfan/digest.js'; +import { redactPii } from '../src/intake/redact.js'; +import { postText, scrub } from '../src/watchdog/post.js'; +import type { Incident } from '../src/watchdog/incidents.js'; +import { mergePostText } from '../src/merge-posts.js'; +import { mainWatchPost } from '../src/intake/channel.js'; +import { readServiceLog, startDashboard } from '../src/dashboard.js'; +import * as doctorModule from '../src/doctor.js'; +import { projectStateDir } from '../src/guardrails/context.js'; +import { loadConfig } from '../src/config/load.js'; +import { safeLine } from '../src/gh-login.js'; +import { scrub as benchScrub } from '../scripts/bench/cli.js'; +import { BRAND } from '../src/brand.js'; +import { exampleProject, exampleRoles, repoRoot } from './helpers.js'; +import { leaks, run, SHAPES, type StandIn } from './token-standins.js'; + +const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); +/** The root updater's redactor (scripts/machine/-update.cjs), loaded as the updater is. */ +const updater = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`)) as { redact(text: string): string; redactDeep(v: unknown): unknown }; +/** Each stand-in in a log line, and as an environment assignment. */ +const lines = (s: StandIn) => [`deploy failed: ${s.value} (exit 1)`, `error: env had TOKEN=${s.value}; giving up`]; +/** The stand-ins whose secret part survives `out(s)`, every miss listed. */ +const survivors = async (out: (text: string) => string | Promise) => { + const missed: string[] = []; + for (const s of SHAPES) for (const l of lines(s)) if (leaks(await out(l), s.secret)) missed.push(`${s.what} in "${l.slice(0, 18)}…"`); + return missed; +}; +/** The production composition the coordinator hands the digest and the merge posts (src/coordinator.ts, src/coordinator/superfan.ts). */ +const coordinatorRedact = (s: string) => scrub(redactPii(s, { names: [], patterns: [] })); +const pem = () => SHAPES.find((s) => s.kind === 'private key')!; + +test('dashboard: a private key logged one journal entry per line is redacted whole, and the entries around it stay', () => { + const msgs = ['starting', ...pem().value.split('\n'), 'done']; + const journal = () => ({ status: 0, stderr: '', stdout: msgs.map((m, i) => JSON.stringify({ MESSAGE: m, PRIORITY: '6', __REALTIME_TIMESTAMP: String(1_767_225_600_000_000 + i) })).join('\n') }); + const shown = readServiceLog({ unit: 'coordinator.service', file: null }, 10, journal).entries.map((e) => e.message); + assert.ok(!leaks(JSON.stringify(shown), pem().secret), JSON.stringify(shown)); + assert.deepEqual([shown[0], shown.at(-1)], ['starting', 'done']); +}); + +test('dashboard: a private key on one long journal line (over the 2,000-character cut) is redacted before it is cut', () => { + const body = run(3000); + const key = JSON.stringify(['-----BEGIN RSA PRIVATE KEY-----', body, '-----END RSA PRIVATE KEY-----'].join('\n')); + const journal = () => ({ status: 0, stderr: '', stdout: JSON.stringify({ MESSAGE: `config: ${key}`, PRIORITY: '3', __REALTIME_TIMESTAMP: '1767225600000000' }) }); + const shown = JSON.stringify(readServiceLog({ unit: 'coordinator.service', file: null }, 10, journal).entries); + assert.ok(!leaks(shown, body), shown.slice(0, 300)); + assert.ok(!leaks(redactString(`config: ${key}`), body), 'and redactString alone'); +}); + +test('redactRecords: records come back as many as went in, each with its own lines; a key over several is found', () => { + const records = ['a', 'b\nc', ...pem().value.split('\n'), 'd']; + assert.equal(typeof redactModule.redactRecords, 'function', 'the shared redactor redacts records as one text'); + const out = redactModule.redactRecords(records); + assert.equal(out.length, records.length); + assert.deepEqual([out[0], out[1], out.at(-1)], ['a', 'b\nc', 'd']); + assert.ok(!leaks(out.join('\n'), pem().secret)); +}); + +test('the shapes are found after an underscore too (`MY_`), and a lowercase bearer', () => { + const missed: string[] = []; + for (const s of SHAPES.filter((x) => x.token)) { + for (const before of ['x_', 'MY_TOKEN_', 'a_b_']) if (leaks(redactString(`${before}${s.value} end`), s.secret)) missed.push(`${s.what} after ${before}`); + } + assert.deepEqual(missed, []); + for (const scheme of ['bearer', 'BEARER', 'Bearer']) assert.equal(redactString(`authorization: ${scheme} ${run(40)}`), `authorization: ${scheme} [REDACTED]`); + // Ordinary snake_case and the like are left alone. + for (const s of ['my_sk-ish_name', 'the_0x_prefix', 'a_eyJ_note', 'config_AKIA_note', 'auth_bearer_header']) assert.equal(redactString(s), s); +}); + +test('gh-login, the bench and the root updater: no stand-in survives their own output (each goes through the shared redactor)', async () => { + assert.deepEqual(await survivors((t) => safeLine(t)), [], "gh's output line on the dashboard"); + assert.deepEqual(await survivors((t) => benchScrub(t, {})), [], "the bench's printed line"); + assert.deepEqual(await survivors((t) => JSON.stringify(updater.redactDeep({ reason: t }))), [], "the updater's log line"); +}); + +/** + * What each module's own rules masked before #889 (gh-login's TOKENISH, the bench's TOKEN_SHAPE, the updater's + * redact()): a token cut short, a clone URL's user info, a value after a secret's name, a short bearer token. Each + * module masks the same now, through the shared redactor's broad rules, as its own mask word. + */ +const CUT_SHORT: [string, string, string][] = [ + ['a GitHub token of 8', `error: ${'gho'}_${run(8)} rejected`, run(8)], + ['a fine-grained token of 16', `x ${'github'}_pat_11${run(14)} y`, run(14)], + ['an Anthropic key of 3', `key ${'sk'}-ant-${run(3, 'Qz9')} rejected`, `-ant-${run(3, 'Qz9')}`], + ['a Slack token of 2', `with ${'xoxb'}-${run(2, 'Z9')} now`, `-${run(2, 'Z9')} now`], + ['a JWT after a hyphen', `x-eyJ${run(10)}.${run(10, 'pQ7')}.${run(10, 'mN4')} y`, run(10, 'mN4')], + ['a short bearer token', `Authorization: bearer ${run(10, 'bR5')}`, run(10, 'bR5')], + ['a clone URL with user info', `https://someone:${run(12, 'hT2')}@git.example.test/x.git`, run(12, 'hT2')], + ['a value after a secret name', `GH_TOKEN=${run(12, 'vW3')} failed`, run(12, 'vW3')], +]; + +test("the broad rules: what gh-login, the bench and the updater masked on their own is masked through the shared redactor, as each one's word", () => { + const missed: string[] = []; + for (const [what, text, secret] of CUT_SHORT) { + const outs: [string, string][] = [ + ['redactString broad', redactString(text, { broad: true } as Parameters[1])], + ['the bench', benchScrub(text, {})], + ['the updater', updater.redact(text)], + ]; + if (what.startsWith('a GitHub') || what.startsWith('a fine-grained')) outs.push(['gh-login', safeLine(text)]); + for (const [who, out] of outs) if (out.includes(secret)) missed.push(`${what} via ${who}: ${out}`); + } + assert.deepEqual(missed, []); + // Each one's own word, as before. + assert.equal(safeLine(CUT_SHORT[0]![1]), 'error: [hidden] rejected'); + assert.match(updater.redact(CUT_SHORT[6]![1]), /^https:\/\/\[redacted\]@git\.example\.test/); + assert.match(benchScrub(CUT_SHORT[2]![1], {}), /^key \[redacted\] rejected$/); + // A mask with a `$` in it is taken literally. + assert.equal(redactString(`token=${run(12)}`, { broad: true, mask: '[$1]' } as Parameters[1]), 'token=[$1]'); +}); + +test('the broad rules are off by default: ordinary text with secret words in it is left as it was', () => { + for (const s of ['the token: see docs/auth.md', 'password reset, then sign in at https://example.test/login', 'bearer of bad news']) assert.equal(redactString(s), s); +}); + +/** A dashboard on a scratch state dir; `key`: a key file written first. */ +async function dashboard(key?: string) { + const dir = scratch('dash-key-'); + const { dir: root } = exampleProject(); + const web = scratch('dash-key-web-'); + writeFileSync(join(web, 'index.html'), 'ok'); + const db = join(dir, 'events.db'); + new EventLog(db).close(); + if (key !== undefined) { + mkdirSync(join(dir, 'state'), { recursive: true }); + writeFileSync(join(dir, 'state', 'dashboard-token'), key, { mode: 0o600 }); + } + const d = await startDashboard({ root, cfg: loadConfig(root), eventsDb: db, stateDir: join(dir, 'state'), user: 'example-owner', roleOf: exampleRoles, webDir: web, pollMs: 50, slotsDir: join(dir, 'slots') }); + return { d, file: join(dir, 'state', 'dashboard-token') }; +} + +test('dashboard keys: a new key has the prefix and is redacted in every output path; a commit hash, a digest or an id is not', async () => { + const { d } = await dashboard(); + try { + assert.match(d.token, /^wld_[A-Za-z0-9_-]{32}$/, 'a new key: wld_ and 32 base64url characters'); + assert.equal(readFileSync(d.keyFile, 'utf8').trim(), d.token); + assert.equal((await fetch(`${d.url}/api/role`, { headers: { authorization: `Bearer ${d.token}` } })).status, 200); + // The real key, standing alone, through every path the stand-ins go through. + const key: StandIn = { kind: 'dashboard key', what: 'the new key', value: d.token, secret: d.token, token: true }; + const missed: string[] = []; + const log = new EventLog(':memory:'); + const f = new FileBacklog(join(scratch('bl-'), 'b.json'), 'harness'); + const n = f.open({ title: 'x', body: '', author: 'a' }); + const incident = (detail: string): Incident => ({ id: 'i1', kind: 'k', key: 'k', instance: 'alpha', severity: 'red', title: 'T', detail, fix: 'F', tried: [], since: '2026-02-01T00:00:00.000Z', lastSeen: '2026-02-01T00:00:00.000Z', resolvedAt: null, posted: {} }); + const file = join(scratch('svc-log-'), 'c.log'); + const paths: [string, (t: string) => string | Promise][] = [ + ['redactString', (t) => redactString(t)], + ['events', (t) => JSON.stringify(log.append('coordinator.error', { instance: 'a', where: 'w', kind: 'error', message: t }, 'c').payload)], + ['comments', async (t) => (await f.comment(n, t), JSON.stringify((await f.comments(n)).at(-1)))], + ['digest', (t) => buildDigest({ journal: { entries: [{ at: '2026-02-01T00:00:00.000Z', priority: 3, message: t }], problem: null } }, { since: '2026-01-01T00:00:00.000Z', eventId: 0, open: null, health: null }, new Date('2026-02-02T00:00:00.000Z'), coordinatorRedact).items[0]!.line], + ['incident post', (t) => postText(incident(t), 'raised')], + ['merge post', (t) => mergePostText({ number: 7, url: 'https://github.com/o/r/pull/7', title: 'T', body: t, mergedAt: null, mergeCommit: null, mergedBy: 'ada', base: 'main' }, 'a3f1c9e0b2d4', coordinatorRedact)], + ['chat notice', (t) => mainWatchPost({ sha: 'a3f1c9e0b2d4', url: 'https://github.com/o/r/commit/a3f1c9e0b2d4', reason: t, revert: null, no_revert: null }, { dashboardUrl: 'https://dash.example.test' }).text], + ['service log', (t) => (writeFileSync(file, `${t}\n`), JSON.stringify(readServiceLog({ unit: null, file }, 10).entries))], + ['gh-login', (t) => safeLine(t)], + ['bench', (t) => benchScrub(t, {})], + ['updater', (t) => updater.redact(t)], + ]; + for (const [name, out] of paths) for (const l of lines(key)) if (leaks(await out(l), key.secret)) missed.push(`${name}: ${l.slice(0, 14)}…`); + assert.deepEqual(missed, []); + } finally { + await d.close(); + } + // Not redacted: a 32-character hex hash (an MD5, an engine id), a commit sha, a sha256 digest, a UUID. + for (const v of ['a3f1c9e0b2d4f6a8'.repeat(2), 'a3f1c9e0b2d4f6a8c0e2b4d6f8a0c2e4b6d8f0a2', '7e9b5d3c1f0a2468'.repeat(4), '123e4567-e89b-12d3-a456-426614174000']) { + assert.equal(redactString(`commit ${v} built`), `commit ${v} built`, v); + } +}); + +test('dashboard keys: an older key (no prefix) keeps opening the dashboard and is not rewritten', async () => { + const old = run(32, 'Ol9dKeyX_-'); + const { d, file } = await dashboard(old); + try { + assert.equal(d.token, old); + assert.equal((await fetch(`${d.url}/api/role`, { headers: { authorization: `Bearer ${old}` } })).status, 200, 'the older key still opens it'); + assert.equal((await fetch(`${d.url}/api/role`, { headers: { authorization: 'Bearer wrong' } })).status, 401); + assert.equal(readFileSync(file, 'utf8'), old, 'the file is left as it was'); + } finally { + await d.close(); + } +}); + +test('doctor: an older dashboard key asks for rotation; a new one is fine; none, nothing; the key is never shown', () => { + const check = doctorModule.dashboardKeyCheck; + assert.equal(typeof check, 'function', 'doctor has a dashboard key check'); + const dir = scratch('doctor-key-'); + const old = run(32, 'Ol9dKeyX_-'); + writeFileSync(join(dir, 'old'), `${old}\n`); + writeFileSync(join(dir, 'new'), `wld_${run(32, 'Nw8-k_')}`); + const warned = check(join(dir, 'old'), 'local')!; + assert.equal(warned.level, 'warn'); + assert.match(warned.detail, /older format[\s\S]*Rotate it/); + assert.ok(!warned.detail.includes(old), 'the key is not shown'); + assert.equal(check(join(dir, 'new'), 'local')!.level, 'ok'); + assert.equal(check(join(dir, 'none'), 'local'), null); + // In doctor's own run, for the project's local dashboard. + const { dir: root } = exampleProject(); + const state = projectStateDir(root); + mkdirSync(state, { recursive: true }); + writeFileSync(join(state, 'dashboard-token'), old, { mode: 0o600 }); + const all = doctorModule.doctor(root); + assert.equal(all.find((c) => c.name === 'dashboard key local')?.level, 'warn'); + assert.ok(!JSON.stringify(all).includes(old), 'doctor never prints the key'); +}); + +test('the dashboard-key and broad patterns, the underscore pass and redactRecords run in linear time on hostile input', () => { + const N = 100_000; + const fill = (unit: string) => unit.repeat(Math.ceil(N / unit.length)); + const hostile = [fill('a-'), fill('a.'), fill('-a'), fill('wld_'), fill('wld_a-'), fill('-wld_'), fill('eyJ-'), fill('x-eyJaaaaaaaaa.'), fill('a+'), fill('a://'), fill('a:/'), fill('token'), fill('token='), fill('token="'), fill(' token'), fill('bearer '), fill('ghp_'), fill('-ghp_a'), fill('sk-ant-'), fill('xoxb-'), fill('_ghp_'), fill('x_sk-'), fill('_'), fill('a_b'), fill('_Bearer ')]; + const slow: string[] = []; + const patterns = REDACTION_PATTERNS.filter((p) => p.kind === 'dashboard key' || p.group === 'broad'); + assert.ok(patterns.length >= 1 + 7, `the dashboard-key pattern and the broad rules are in the shared redactor (${patterns.length})`); + const time = (what: string, f: () => unknown, ms = 300) => { + const t0 = performance.now(); + f(); + const took = performance.now() - t0; + if (took > ms) slow.push(`${what}: ${Math.round(took)} ms`); + }; + for (const { kind, re, rep } of patterns) for (const s of hostile) time(`${kind} on ${s.slice(0, 12)}…`, () => (typeof rep === 'string' ? s.replace(re, rep) : s.replace(re, rep))); + // The whole redactor, the underscore pass and the broad rules included, and records joined as one text. + for (const s of [fill('_ghp_'), fill('x_sk-'), fill('a_'), fill('_eyJ.')]) time(`redactString on ${s.slice(0, 12)}…`, () => redactString(s, { broad: true } as Parameters[1]), 1500); + time('redactRecords on 20,000 records', () => redactModule.redactRecords?.(Array.from({ length: 20_000 }, (_, i) => `line ${i} -----BEGIN`)), 1500); + assert.deepEqual(slow, []); +}); diff --git a/test/redact-shapes.test.ts b/test/redact-shapes.test.ts index cf6f56260..dda8fbae1 100644 --- a/test/redact-shapes.test.ts +++ b/test/redact-shapes.test.ts @@ -2,6 +2,7 @@ // path redacts through it. Each stand-in in test/token-standins.ts goes through redactString, and through each path // that stores or sends text: the event log, issue comments (file and GitHub backlogs), the superfan's digest, // the notifications (an incident post, a merge post, a chat notice) and the dashboard's service-log view. +// Round 2 (dashboard keys, the broad rules, the journal view, glued shapes): test/redact-records-keys.test.ts. import './helpers.js'; import { test } from 'node:test'; import assert from 'node:assert/strict'; @@ -20,7 +21,7 @@ import type { Incident } from '../src/watchdog/incidents.js'; import { mergePostText } from '../src/merge-posts.js'; import { mainWatchPost } from '../src/intake/channel.js'; import { readServiceLog } from '../src/dashboard.js'; -import { leaks, run, SHAPES, type StandIn } from './token-standins.js'; +import { leaks, SHAPES, type StandIn } from './token-standins.js'; const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); /** Each stand-in in a log line, and as an environment assignment. */ @@ -35,10 +36,10 @@ const survivors = async (out: (text: string) => string | Promise) => { const coordinatorRedact = (s: string) => scrub(redactPii(s, { names: [], patterns: [] })); test('every secret pattern of the shared redactor has a stand-in, and secretKinds names each stand-in by its kind', () => { - const kinds = [...new Set(REDACTION_PATTERNS.filter((p) => p.secret).map((p) => p.kind))]; + const kinds = [...new Set(REDACTION_PATTERNS.filter((p) => p.group === 'secret').map((p) => p.kind))]; assert.deepEqual(kinds.filter((k) => !SHAPES.some((s) => s.kind === k)), [], 'add a stand-in to test/token-standins.ts for each new pattern'); assert.deepEqual(SHAPES.filter((s) => !secretKinds(s.value).includes(s.kind)).map((s) => s.what), []); - for (const k of ['GitHub token', 'API key (sk-)', 'Discord bot token', 'Slack token', 'Slack app-level token', 'Slack webhook', 'Linear key', 'Tailscale key']) { + for (const k of ['GitHub token', 'API key (sk-)', 'Discord bot token', 'Slack token', 'Slack app-level token', 'Slack webhook', 'Linear key', 'Tailscale key', 'dashboard key']) { assert.ok(kinds.includes(k), `the shared redactor knows ${k}`); } }); @@ -150,13 +151,6 @@ test('dashboard: a private key that starts above the kept lines of the log file assert.equal(shown.at(-1)!.message, 'line 7'); }); -test('not covered, by decision: a bare 32-character key with no prefix (a commit hash, a digest or an id looks the same); in a Bearer header it is redacted', () => { - const bare = run(32); - const hex = 'a3f1c9e0b2d4f6a8'.repeat(2); - for (const v of [bare, hex]) assert.equal(redactString(`key ${v} rotated`), `key ${v} rotated`); - assert.ok(!redactString(`Authorization: Bearer ${bare}`).includes(bare)); -}); - test('Discord and Tailscale patterns run in linear time on hostile input', () => { const N = 100_000; const fill = (unit: string) => unit.repeat(Math.ceil(N / unit.length)); diff --git a/test/redact-vendored.test.ts b/test/redact-vendored.test.ts new file mode 100644 index 000000000..3af85c99f --- /dev/null +++ b/test/redact-vendored.test.ts @@ -0,0 +1,42 @@ +// #889: the root updater (scripts/machine/-update.cjs) is one standalone CommonJS file, so it carries the shared +// redactor as a generated copy (scripts/lib/redact-vendor.mjs). The copy is exactly src/events/redact.ts transpiled, +// and the updater's redact() goes through it: the same patterns, in the same order, with the same results. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { BRAND } from '../src/brand.js'; +import { REDACTION_PATTERNS, redactString } from '../src/events/redact.js'; +import { repoRoot } from './helpers.js'; +import { SHAPES } from './token-standins.js'; + +const UPDATER = join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`); +type Lib = { vendoredBlock(source: string): string; blockIn(text: string): string | null }; +const vendor = async () => (await import(pathToFileURL(join(repoRoot, 'scripts', 'lib', 'redact-vendor.mjs')).href)) as Lib; + +test("the updater's copy of the shared redactor is src/events/redact.ts, transpiled, byte for byte", async () => { + const lib = await vendor(); + const block = lib.blockIn(readFileSync(UPDATER, 'utf8')); + assert.ok(block, 'the updater has the generated block'); + assert.equal(block, lib.vendoredBlock(readFileSync(join(repoRoot, 'src', 'events', 'redact.ts'), 'utf8')), 'out of date: run node scripts/gen-redact-vendor.mjs --write'); +}); + +test("the updater's redact() masks every shape the shared redactor knows, as the shared redactor does, plus its broad rules", () => { + const u = createRequire(import.meta.url)(UPDATER) as { redact(text: string): string }; + const shared = (t: string) => redactString(t, { broad: true, mask: '[redacted]' } as Parameters[1]); + for (const s of SHAPES) { + for (const line of [`deploy failed: ${s.value} (exit 1)`, `GH_TOKEN=${s.value}`, `see https://someone:${s.secret}@git.example.test/x`]) assert.equal(u.redact(line), shared(line), s.what); + } + assert.ok(REDACTION_PATTERNS.length > 0); +}); + +test('the updater stays standalone: it requires only node: modules, and its copy requires nothing', async () => { + const lib = await vendor(); + const src = readFileSync(UPDATER, 'utf8'); + const required = [...src.matchAll(/require\(\s*['"]([^'"]+)['"]\s*\)/g)].map((m) => m[1]!); + assert.deepEqual(required.filter((r) => !r.startsWith('node:')), []); + assert.ok(!/\brequire\(|\bimport\(/.test(lib.blockIn(src)!), 'the generated block loads nothing'); +}); diff --git a/test/redaction-one-module.test.ts b/test/redaction-one-module.test.ts index efc782fb7..901f2d542 100644 --- a/test/redaction-one-module.test.ts +++ b/test/redaction-one-module.test.ts @@ -19,7 +19,6 @@ import assert from 'node:assert/strict'; import { readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import ts from 'typescript'; -import { BRAND } from '../src/brand.js'; import { repoRoot } from './helpers.js'; import { SHAPES } from './token-standins.js'; @@ -31,16 +30,6 @@ const CALLS = new Set(['replace', 'replaceAll', 'split']); const BEFORE = 'deploy failed:'; const AFTER = '(exit 1)'; -/** - * Own redactors found today and not moved by #889, each with why: still found (so the list only shrinks), and no other - * one passes. Adding an entry is a loosening (registered never-grow). - */ -const KNOWN_OWN_REDACTORS: readonly { path: string; why: string }[] = [ - { path: 'scripts/bench/cli.ts', why: "the bench's scrub: its TOKEN_SHAPE is broader than the shared shapes (a GitHub token from 16 characters, any sk-ant- or xox- run), for a token cut short; redactString alone would narrow it" }, - { path: `scripts/machine/${BRAND.cli}-update.cjs`, why: "the root updater's journal redact(): a standalone CommonJS script that loads no engine module; broader than the shared shapes (GitHub from 20 characters, sk-ant- from 10)" }, - { path: 'src/gh-login.ts', why: "safeLine masks a GitHub token from 8 characters (gh's output may cut one short), as [hidden]; redactString alone would narrow it" }, -]; - /** A regex from a literal's text (`/a+/gi`), or null when it doesn't compile here. */ function literal(text: string): RegExp | null { const end = text.lastIndexOf('/'); @@ -176,7 +165,6 @@ test('no module but src/events/redact.ts redacts a token shape with its own rege const texts = new Map(files.map((f) => [f, readFileSync(join(repoRoot, f), 'utf8')])); for (const [f, text] of texts) if (f !== SHARED) for (const [k, v] of bindings(parse(f, text))) global.set(k, v); const found = files.filter((f) => f !== SHARED).flatMap((f) => ownRedactors(f, texts.get(f)!, global).map((x) => `${f}:${x}`)); - const known = new Set(KNOWN_OWN_REDACTORS.map((k) => k.path)); - assert.deepEqual(found.filter((x) => !known.has(x.slice(0, x.indexOf(':')))), [], 'add the shape to PATTERNS in src/events/redact.ts and call redactString'); - for (const k of KNOWN_OWN_REDACTORS) assert.ok(found.some((x) => x.startsWith(`${k.path}:`)), `${k.path} no longer has its own redactor: take it out of KNOWN_OWN_REDACTORS`); + // No exception: a module whose text may cut a token short uses the shared redactor's broad rules (`broad: true`). + assert.deepEqual(found, [], 'add the shape to PATTERNS (or BROAD) in src/events/redact.ts and call redactString'); }); diff --git a/test/token-standins.ts b/test/token-standins.ts index 6e18833eb..e2915c84b 100644 --- a/test/token-standins.ts +++ b/test/token-standins.ts @@ -74,6 +74,9 @@ export const SHAPES: readonly StandIn[] = [ t('Tailscale key', 'a Tailscale auth key', `tskey-auth-k${upper(10)}CNTRL-${run(33)}`), t('Tailscale key', 'a Tailscale API token', `tskey-api-k${upper(10)}CNTRL-${run(33)}`), t('Tailscale key', 'a Tailscale client secret', `tskey-client-k${upper(10)}CNTRL-${run(33)}`), + // A dashboard key (#889: prefixed, so it is found standing alone), with a `-` and a `_` as base64url has. The prefix + // is written out: keys already issued carry it, so changing it is a change this list must see. + t('dashboard key', 'a dashboard key', `${j('_', 'wld', run(30, 'kQ9-zX_7mP2v'))}Ab`), // The rest of the shapes the shared redactor knows. t('Stripe key', 'a Stripe live key', j('_', 'sk', 'live', run(24))), t('Stripe key', 'a Stripe restricted key', j('_', 'rk', 'live', run(24))), From 7bfb696c621b63bcf1f43e3b969fda63b45b3c50 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 23:58:31 -0500 Subject: [PATCH 4/9] redaction: broad mode masks the union of every rule's spans, private keys fail closed, glue class (#889) Round 3, on the second adversarial review and the round-2 final check (#921): - The broad mode (the root updater, gh-login, the bench) no longer applies rules in turn: every rule runs on the text as it was, again with boundaries at the glue points and where a span ends between word characters, and the union is masked once, read again on the result until it settles. The updater's own token rule on main is one of the rules; the URL user-info rule takes a scheme of up to 32 characters from any start; a JWT takes its further dotted parts. On main's three module rules (fixtures in the test), generated input shows no secret more often than before (the regression the final check measured). - Glue class: a token right after an ANSI escape or a JSON backslash escape is found, by a second read with a boundary marked after each (default mode too). - Private keys fail closed: no END redacts to the end of the text, no BEGIN from the start; PGP's PRIVATE KEY BLOCK too. - HTTP Basic credentials are redacted; a dashboard key after `-` or a letter and a token followed by `_` are found. - decision-detail and chat-attachments redact before they cut; doctor checks the hub's key. - The bench drops format characters before redacting and keeps its control characters for after (the redactor finds ANSI glue); its email rule reads a mask inside an address. Test (fails without): test/redact-fail-closed.test.ts and test/redact-broad-union.test.ts, 11 of 14 failing at 8633356d. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 9 +- scripts/bench/cli.ts | 17 +- scripts/machine/worklane-update.cjs | 208 ++++++++++++++++--- src/chat-attachments.ts | 3 +- src/decision-detail.ts | 6 +- src/doctor.ts | 16 +- src/events/redact.ts | 270 +++++++++++++++++++++---- src/merge-policy-weakening.ts | 1 + test/redact-broad-union.test.ts | 180 +++++++++++++++++ test/redact-fail-closed.test.ts | 163 +++++++++++++++ test/token-standins.ts | 3 + 11 files changed, 793 insertions(+), 83 deletions(-) create mode 100644 test/redact-broad-union.test.ts create mode 100644 test/redact-fail-closed.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index 287a3bc7a..7ca636d16 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -1,10 +1,11 @@ ### Upgrading (from v0.5.2) -- **Dashboard keys now start with `wld_` (#889).** A new key is `wld_` and 32 random characters, a shape the redactor finds wherever it appears. A key from before keeps working until you rotate it. `doctor` warns on each older key it can read and says how to rotate it: remove the key file and restart what serves that dashboard. On a server with the hub, then run `scripts/setup/dashboards.sh` as root so the hub can read the new key. +- **Dashboard keys now start with `wld_` (#889).** A new key is `wld_` and 32 random characters, a shape the redactor finds wherever it appears. A key from before keeps working until you rotate it. `doctor` warns on each older key it can read (the local dashboard's, each instance's and the hub's) and says how to rotate it: remove the key file and restart what serves that dashboard. On a server with the hub, then run `scripts/setup/dashboards.sh` as root so the hub can read the new key. ### Guardrail changes: what each one does -- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key and a dashboard key, and finds a token glued to a name after `_` (`MY_ghp_...`) and a lowercase `bearer`. Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. -- **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad rules (a token cut short, a clone URL's user info, a value after a secret's name, a short bearer token) keep everything those modules masked before. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. -- **The dashboard's service log** redacts its journal entries and log-file lines as one text, so a private key logged a line at a time is masked whole. It redacts a long line before cutting it to 2,000 characters. +- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. It also finds a token glued to a name after `_` (`MY_ghp_...`), right after an ANSI colour code or a JSON escape (`\n`), and after a lowercase `bearer`. Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. +- **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. PGP's private key block is covered too. A key whose BEGIN and END are both outside a window (only its middle lines in view) is not recognised. +- **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. +- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section and a chat attachment are redacted before they are cut, too. - A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/bench/cli.ts b/scripts/bench/cli.ts index f59ed3112..1b80e0052 100644 --- a/scripts/bench/cli.ts +++ b/scripts/bench/cli.ts @@ -75,8 +75,11 @@ export interface Check { /** A variable whose value may be a credential (container.ts SECRET_NAME without the daemon's names, which are paths). */ const SECRET_VAR = /KEY|TOKEN|SECRET|PASSW|CREDENTIAL|AUTH|COOKIE|SESSION/i; -/** An email address: its domain ends in a letters-only label, so a package spec such as `name@0.0.79` is not one. */ -const EMAIL = /[\w.+-]+@[\w-]+(?:\.[\w-]+)*\.[A-Za-z]{2,}\b/g; +/** + * An email address: its domain ends in a letters-only label, so a package spec such as `name@0.0.79` is not one. Read + * after the token masks, so a mask may stand in a label (`x@wld_[redacted].com`): the address is still found. + */ +const EMAIL = /(?:[\w.+-]|\[redacted\])+@(?:[\w-]|\[redacted\])+(?:\.(?:[\w-]|\[redacted\])+)*\.[A-Za-z]{2,}\b/g; /** What scrub() keeps of one detail, and of one printed line. */ export const DETAIL_MAX = 400; const LINE_MAX = 1000; @@ -90,7 +93,8 @@ const PATH_VALUE = /^(\/|unix:)/; * or switch the login (PIECE_MIN characters or more), and every piece of such a value of PIECE_MIN or more that starts * or ends it (#851 round 2: a message cut short upstream, as the driver's checks cut a child's stderr, ends in a * prefix of the value; this redacts it whatever cut it); every token-shaped string. Email addresses become [email]; - * control and format characters (an ANSI escape's ESC, a bell, a bidi override) a space. Cut to at most `max` + * every control character (an ANSI escape's ESC, a bell, a line break) a space; a format character (a zero-width + * space, a bidi override) is dropped, before the token shapes are looked for. Cut to at most `max` * characters only after all of that. * * INVARIANT (#884 round 2): first of all, every word shaped like a task is withheld (withholdTasks), on the text as it @@ -111,8 +115,11 @@ export function scrub(text: string, env: NodeJS.ProcessEnv, max = DETAIL_MAX): s for (const piece of [v.slice(0, n), v.slice(v.length - n)]) if (s.includes(piece)) s = s.split(piece).join('[redacted]'); } } - // Token shapes through the shared redactor (#889), its broad rules too (a token cut short, a value after a secret's name). - s = redactString(s, { broad: true, mask: '[redacted]' }).replace(EMAIL, '[email]').replace(/[\p{Cc}\p{Cf}]/gu, ' '); + // A format character (a zero-width space, a bidi override) is dropped before the token shapes are looked for, so it + // can't split one (#889 round 3). Then the shared redactor, its broad rules too (a token cut short, a value after a + // secret's name); it finds a token right after an ANSI escape itself (its glue points). Then emails, as before, and + // every control character becomes a space. + s = redactString(s.replace(/\p{Cf}/gu, ''), { broad: true, mask: '[redacted]' }).replace(EMAIL, '[email]').replace(/\p{Cc}/gu, ' '); return s.length > max ? `${s.slice(0, max - 3)}...` : s; } diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 105fa7edf..4ae34b13f 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -121,6 +121,7 @@ exports.redactString = redactString; exports.redactRecords = redactRecords; exports.redact = redact; const credentialShaped = (v) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); +const basicShaped = (v) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)); const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'jwt', 'bearer', 'credential', 'credentials', 'authorization']); const CREDENTIAL_ENDINGS = ['token', 'secret', 'password', 'passwd', 'accesskey']; const credentialWord = (w) => CREDENTIAL_WORDS.has(w) || CREDENTIAL_ENDINGS.some((e) => w.endsWith(e)); @@ -144,24 +145,26 @@ const DASHBOARD_KEY_BODY = '[A-Za-z0-9_-]{32,}'; const isDashboardKey = (key) => new RegExp(`^${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}$`).test(key); exports.isDashboardKey = isDashboardKey; const PATTERNS = [ - [new RegExp(`(^|[^A-Za-z0-9_-])${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `$1${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], - [/\b(gh[pousr]_[A-Za-z0-9]{30,})\b/g, 'gh*_[REDACTED]', 'GitHub token'], + [new RegExp(`${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], + [/\b(gh[pousr]_[A-Za-z0-9]{30,})(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]', 'Google API key'], - [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*/g, '$1[REDACTED JWT]', 'JWT'], - [/\b(AKIA|ASIA)[A-Z0-9]{16}\b/g, 'AKIA[REDACTED]', 'AWS access key'], - [/\bxoxe[.-][A-Za-z0-9.-]{10,}\b/g, 'xoxe-[REDACTED]', 'Slack token'], - [/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g, 'xox-[REDACTED]', 'Slack token'], - [/\bxapp-[A-Za-z0-9-]{10,}\b/g, 'xapp-[REDACTED]', 'Slack app-level token'], - [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}\b/g, 'lin_[REDACTED]', 'Linear key'], + [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], + [/\b(AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], + [/\bxoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], + [/\bxox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], + [/\bxapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], + [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], - [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]', 'private key'], + [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], + [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], + [/\b(Basic|basic|BASIC)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/g, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], - [/\b0x[0-9a-fA-F]{64}\b/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], + [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], [ /(\\?")([A-Za-z0-9_-]+)\1(\s*:\s*)\1([A-Za-z0-9_\-.+/=~]{16,})\1/g, (m, q, k, sep, v) => (credentialKey(k) && credentialShaped(v) ? `${q}${k}${q}${sep}${q}[REDACTED]${q}` : m), @@ -172,10 +175,11 @@ const PATTERNS = [ const BROAD = [ [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], + [/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, (k) => k, "the updater's token rule"], [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], - [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => `$1${k}`, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], - [/(^|[^a-z0-9+.-])([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, (k) => `$1$2${k}@`, 'user info in a URL'], + [/([a-z][a-z0-9+.-]{0,31}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], ]; const CODE_HEADS = new Set(['this', 'self', 'super', 'opts', 'options', 'o', 'ctx', 'context', 'config', 'cfg', 'conf', 'settings', 'props', 'state', 'env', 'process', 'module', 'exports', 'window', 'document', 'globalThis', 'args', 'params', 'req', 'res', 'request', 'response', 'obj', 'result', 'item', 'e', 'err', 'x']); @@ -212,21 +216,45 @@ exports.REDACTION_PATTERNS = [ ...BROAD.map(([re, rep, kind]) => ({ kind, re, rep: rep('[REDACTED]'), group: 'broad' })), ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'identifier' })), ]; -const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', exports.DASHBOARD_KEY_PREFIX]; -function afterUnderscore(s, f) { - const cuts = []; +const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', exports.DASHBOARD_KEY_PREFIX]; +const GLUE = /\u001b(?:\[[0-?]*[ -/]*[@-~]|[@-Z\\-_])|\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])/g; +function gluePoints(s) { + const cuts = [...s.matchAll(GLUE)].map((m) => m.index + m[0].length); for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) if (GLUED_PREFIXES.some((p) => s.startsWith(p, at + 1))) cuts.push(at + 1); + return [...new Set(cuts)].filter((c) => c < s.length).sort((a, b) => a - b); +} +function freeMark(s) { + const held = new Set(); + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xe000 && c <= 0xf8ff) + held.add(c); + } + for (let c = 0xe000; c <= 0xf8ff; c++) + if (!held.has(c)) + return String.fromCharCode(c); + return null; +} +function glued(s, f) { + const cuts = gluePoints(s); if (!cuts.length) return s; - let out = ''; + const mark = freeMark(s); + if (mark === null) { + let out = s.slice(0, cuts[0]); + for (const [i, c] of cuts.entries()) + out += f(s.slice(c, cuts[i + 1] ?? s.length)); + return out; + } + let marked = ''; let from = 0; - for (const c of [...cuts, s.length]) { - out += from === 0 ? s.slice(0, c) : f(s.slice(from, c)); + for (const c of cuts) { + marked += s.slice(from, c) + mark; from = c; } - return out; + return f(marked + s.slice(from)).split(mark).join(''); } const expand = (rep, m, groups) => rep.replace(/\$(\d|\$|&)/g, (_, k) => (k === '$' ? '$' : k === '&' ? m : String(groups[Number(k) - 1] ?? ''))); const breaks = (s) => s.split('\n').length - 1; @@ -281,19 +309,139 @@ function secretKinds(s) { } return [...kinds].sort(); } +const globals = new Map(); +const globalOf = (re) => { + let g = globals.get(re); + if (!g) + globals.set(re, (g = re.flags.includes('g') ? re : new RegExp(re.source, `${re.flags}g`))); + return g; +}; +function spansOf(t, re, rep, out) { + const g = globalOf(re); + g.lastIndex = 0; + for (let m = g.exec(t); m; m = g.exec(t)) { + const whole = m[0]; + g.lastIndex = Math.max(m.index + 1, m.index + whole.length - 64); + const groups = m.slice(1).map((x) => x ?? ''); + const r = typeof rep === 'string' ? expand(rep, whole, groups) : rep(whole, ...groups); + if (r === whole) + continue; + let a = 0; + while (a < whole.length && a < r.length && whole[a] === r[a]) + a++; + let b = 0; + while (b < whole.length - a && b < r.length - a && whole[whole.length - 1 - b] === r[r.length - 1 - b]) + b++; + if (whole.length - b > a) + out.push([m.index + a, m.index + whole.length - b]); + } +} +const BROAD_RULES = [...PATTERNS.map(([re, rep]) => [re, rep]), ...BROAD.map(([re, rep]) => [re, rep('[REDACTED]')])]; +function maskOnce(s, mask, keepLines) { + const spans = []; + for (const [re, rep] of BROAD_RULES) + spansOf(s, re, rep, spans); + const word = (i) => /[A-Za-z0-9_]/.test(s[i] ?? ''); + readAt(s, gluePoints(s), spans); + readAt(s, [...new Set(spans.flat())].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), spans); + const masks = []; + if (mask) + for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) + masks.push([at, at + mask.length]); + const kept = masks.length ? spans.flatMap(([a, b]) => outside([a, b], masks)) : spans; + if (!kept.length) + return s; + kept.sort((x, y) => x[0] - y[0]); + let out = ''; + let at = 0; + let open = null; + const close = () => { + if (!open) + return; + out += s.slice(at, open[0]) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(open[0], open[1]))) : ''); + at = open[1]; + }; + for (const [a, b] of kept) { + if (b <= a) + continue; + if (open && a <= open[1]) + open[1] = Math.max(open[1], b); + else { + close(); + open = [a, b]; + } + } + close(); + return out + s.slice(at); +} +function readAt(s, cuts, spans) { + if (!cuts.length) + return; + const markChar = freeMark(s); + if (markChar === null) { + for (const [i, c] of cuts.entries()) { + const piece = s.slice(c, cuts[i + 1] ?? s.length); + const more = []; + for (const [re, rep] of BROAD_RULES) + spansOf(piece, re, rep, more); + for (const [a, b] of more) + spans.push([a + c, b + c]); + } + return; + } + let marked = ''; + let from = 0; + for (const c of cuts) { + marked += s.slice(from, c) + markChar; + from = c; + } + marked += s.slice(from); + const back = (p) => { + let lo = 0; + let hi = cuts.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (cuts[mid] + mid < p) + lo = mid + 1; + else + hi = mid; + } + return p - lo; + }; + const more = []; + for (const [re, rep] of BROAD_RULES) + spansOf(marked, re, rep, more); + for (const [a, b] of more) + spans.push([back(a), back(b)]); +} +function outside([a, b], regions) { + const out = []; + let from = a; + for (const [x, y] of regions) { + if (y <= from || x >= b) + continue; + if (x > from) + out.push([from, x]); + from = Math.max(from, y); + } + if (from < b) + out.push([from, b]); + return out; +} +function maskUnion(s, mask, keepLines) { + let out = maskOnce(s, mask, keepLines); + for (let i = 0; i < 2; i++) { + const again = maskOnce(out, mask, keepLines); + if (again === out) + break; + out = again; + } + return out; +} function redactString(s, o = {}) { const put = o.keepLines ? applyKeepingLines : apply; - let out = s; - if (o.broad) { - for (const [re, rep, kind] of PATTERNS) - if (kind === 'private key') - out = put(out, re, rep); - const mask = (o.mask ?? '[REDACTED]').replace(/\$/g, '$$$$'); - for (const [re, rep] of BROAD) - out = put(out, re, rep(mask)); - } - const secrets = (t) => PATTERNS.reduce((x, [re, rep]) => put(x, re, rep), t); - out = afterUnderscore(secrets(out), secrets); + const secrets = (t) => PATTERNS.reduce((y, [re, rep]) => put(y, re, rep), t); + let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : glued(secrets(s), secrets); if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); return out; diff --git a/src/chat-attachments.ts b/src/chat-attachments.ts index b479baa28..f23365c9f 100644 --- a/src/chat-attachments.ts +++ b/src/chat-attachments.ts @@ -370,7 +370,8 @@ export function attachmentPrompt(stateDir: string, ids: string[], bundleDir: str } else { const raw = attachmentBytes(stateDir, id)!.toString('utf8'); const room = Math.max(0, Math.min(INLINE_PER_FILE, budget)); - const body = redactString(raw.slice(0, room)); + // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. + const body = redactString(raw).slice(0, room); budget -= body.length; const cut = raw.length > room ? `\n(cut: the first ${room} of ${raw.length} characters)` : ''; lines.push(`${what}:`, `<<>>`, body, `<<>>${cut}`); diff --git a/src/decision-detail.ts b/src/decision-detail.ts index 141679d70..f040581fa 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -143,7 +143,8 @@ export function prSections(body: string | null | undefined): { failWithout: stri const h = heads[i]!; const end = heads.slice(i + 1).find((n) => n.level <= h.level)?.at ?? lines.length; const s = lines.slice(h.at + 1, end).join('\n').trim(); - return s ? redactString(s.slice(0, 20_000)) : null; + // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. + return s ? redactString(s).slice(0, 20_000) : null; }; return { failWithout: section(/fail[- ]?without/i), never: section(/\bnever\b/i) }; } @@ -352,7 +353,8 @@ export async function decisionDiff(root: string, base: string, head: string): Pr execFile('git', ['diff', '--no-color', '--no-ext-diff', `${base}..${head}`], { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { if (err) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); const truncated = stdout.length > MAX; - done({ diff: redactString(truncated ? stdout.slice(0, MAX) : stdout), truncated }); + // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. + done({ diff: redactString(stdout).slice(0, MAX), truncated }); }); }); } diff --git a/src/doctor.ts b/src/doctor.ts index 7d4c20bfc..166f5ae4a 100644 --- a/src/doctor.ts +++ b/src/doctor.ts @@ -16,7 +16,8 @@ import { GIT_HOOK_MARKER, legacyHookChecks } from './install.js'; import { instancesDir, loadInstance, policiesForRepo, policyViolations, readPolicy } from './instance.js'; import { instanceTargets, planMigrations, projectConfigFiles, type Targets } from './migrations.js'; import { projectStateDir } from './guardrails/context.js'; -import { canRunAgents, detectOs, diskFree, fileAccess, nvidiaNodes, shimsNeedShell, userUid, wslAvailable } from './os/index.js'; +import { canRunAgents, detectOs, diskFree, fileAccess, homeOf, nvidiaNodes, shimsNeedShell, stateDir, userUid, wslAvailable } from './os/index.js'; +import { VIEWER_USER } from './setup/app-manifest.js'; import { diskFloorCheck } from './governor-holds.js'; import { policyRootChecks } from './doctor-policy-root.js'; import { findTool, toolBin } from './tool-path.js'; @@ -258,9 +259,16 @@ export function doctor(rootArg: string, opts: { agentshield?: boolean; /** A pol if (cfg?.tests.test_run) out.push(networkCheck()); if (opts.agentshield) out.push(agentShield(root)); out.push(...policyRootChecks()); - // #889: each dashboard key this user can read, the local one and each instance's: an older key asks for rotation. - for (const [where, dir] of [['local', projectStateDir(root)], ...instances.map((n) => [n, join(instancesDir(), n, 'state')])] as [string, string][]) { - const k = dashboardKeyCheck(join(dir, 'dashboard-token'), where); + // #889: each dashboard key this user can read: the local one, each instance's and the hub's (its own user's state dir, + // or the viewer user's when doctor runs as root): an older key asks for rotation. + const viewerHome = homeOf(VIEWER_USER); + const keyDirs: [string, string][] = [['local', projectStateDir(root)], ...instances.map((n): [string, string] => [n, join(instancesDir(), n, 'state')]), ['hub', join(stateDir(), 'dashboard-hub')], ...(viewerHome ? [['hub', join(viewerHome, '.local', 'state', BRAND.cli, 'dashboard-hub')] as [string, string]] : [])]; + const seen = new Set(); + for (const [where, dir] of keyDirs) { + const file = join(dir, 'dashboard-token'); + if (seen.has(file)) continue; + seen.add(file); + const k = dashboardKeyCheck(file, where); if (k) out.push(k); } return out; diff --git a/src/events/redact.ts b/src/events/redact.ts index a816afc4d..3634b4753 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -14,6 +14,12 @@ type Replacement = string | ((match: string, ...groups: string[]) => string); /** Credential-shaped: mixed case with a digit (a random key), or a long hex run (a classic token). */ const credentialShaped = (v: string) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); +/** + * A value after `Basic` that is a credential, not a word: base64 with a digit, `+`, `/` or `=` in it, or a mixed-case + * run of 16 or more characters whose length base64 can have (no padding: not 1 more than a multiple of 4). + */ +const basicShaped = (v: string) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)); + /** A word that makes any key holding it a credential's: `db_password`, `slack_bot_token`, `aws_secret_access_key`. */ const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'jwt', 'bearer', 'credential', 'credentials', 'authorization']); /** @@ -57,9 +63,11 @@ export const isDashboardKey = (key: string): boolean => new RegExp(`^${DASHBOARD // Each pattern with what kind of secret it finds (secretKinds names the kind, never the value). const PATTERNS: [RegExp, Replacement, string][] = [ - // A dashboard key: starts only where a run of key characters starts (`wld_wld_...` is one run), so linear. - [new RegExp(`(^|[^A-Za-z0-9_-])${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `$1${DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], - [/\b(gh[pousr]_[A-Za-z0-9]{30,})\b/g, 'gh*_[REDACTED]', 'GitHub token'], + // A dashboard key, wherever its prefix is (after a `-` or a letter too: `key-wld_...`): a match runs to the end of its + // run of key characters and the next search starts after it, so linear. + [new RegExp(`${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], + // The token's end: no letter or digit after it, so `ghp_..._backup` is found too (a `\b` there needs a non-word). + [/\b(gh[pousr]_[A-Za-z0-9]{30,})(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], // Stripe secret and restricted keys, live and test: random, so with a digit (a long camelCase name after the @@ -68,26 +76,36 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // Google API keys: AIza and 35 more characters. [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]', 'Google API key'], // JSON Web Tokens: a base64url JSON header (so `eyJ`), a payload and a signature (empty when unsigned). - [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*/g, '$1[REDACTED JWT]', 'JWT'], - [/\b(AKIA|ASIA)[A-Z0-9]{16}\b/g, 'AKIA[REDACTED]', 'AWS access key'], + // Any further dotted parts go with it (#889 round 3): a JWE has five, and a token may be followed by more. + [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], + [/\b(AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], // Slack's token-rotation and refresh tokens (xoxe-..., xoxe.xoxb-...), before the // other Slack tokens (one is inside it); then its app-level token (xapp-, Socket Mode). - [/\bxoxe[.-][A-Za-z0-9.-]{10,}\b/g, 'xoxe-[REDACTED]', 'Slack token'], - [/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g, 'xox-[REDACTED]', 'Slack token'], - [/\bxapp-[A-Za-z0-9-]{10,}\b/g, 'xapp-[REDACTED]', 'Slack app-level token'], + [/\bxoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], + [/\bxox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], + [/\bxapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], // Linear personal API keys and OAuth tokens. - [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}\b/g, 'lin_[REDACTED]', 'Linear key'], + [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], // Tailscale auth keys, API tokens and OAuth client secrets (tskey-auth-..., tskey-api-..., tskey-client-...): the // prefix and the rest of its run, however short (src/tailscale-setup.ts redacted them this way on its own). [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], - [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]', 'private key'], + // A private key (PEM, and PGP's PRIVATE KEY BLOCK), failing closed (#889): from its BEGIN line to its END line, or + // to the end of the text when no END follows (a key cut short, or still being written); then, for an END line with + // no BEGIN before it (a key whose start is above a window), from the start of the text to that END line. Each BEGIN + // is read once (its match runs to its END or to the end), and the second rule has one start: linear. + [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], + [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], // The scheme is case-insensitive (RFC 9110): `bearer` and `BEARER` too (#889). [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], + // HTTP Basic credentials (base64 of user:password; git's extraheader, src/dashboard-auth.ts's client id and secret), + // failing closed: any credential-looking value after the scheme. Prose ("Basic Authentication") has no digit, `+`, + // `/` or `=`, and isn't a mixed-case run of a base64 length. + [/\b(Basic|basic|BASIC)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/g, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], // Credentials inside connection strings: keep scheme, user and host. [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], // 0x-prefixed 32-byte hex (EVM private keys and the like). - [/\b0x[0-9a-fA-F]{64}\b/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], + [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], // A JSON string value under a credential key (credentialKey: "aws_secret_access_key", "db_password", "x-api-key", // "slack_bot_token", "authorization", ...), also inside an escaped JSON string (\"token\":\"...\"). Not "cacheKey" // or "sort_key", and only a credential-shaped value (16 or more characters; mixed case with a digit, or long hex), @@ -110,18 +128,24 @@ const PATTERNS: [RegExp, Replacement, string][] = [ /** * The broad rules (`broad: true`), for text whose source may cut a token short or name a secret by its key: gh's * output (src/gh-login.ts), the bench's printed lines (scripts/bench/cli.ts) and the root updater's files - * (scripts/machine/-update.cjs, which carries a generated copy of this module). Applied before PATTERNS, each - * puts `mask` (default [REDACTED]) in place of what it finds. They were those modules' own rules (#889): a known + * (scripts/machine/-update.cjs, which carries a generated copy of this module). Run with PATTERNS, all on the + * text as it was, and what any of them finds is masked with `mask` (maskUnion). They were those modules' own rules (#889): a known * token prefix and a short run, a URL's user info, a value after a secret's name, a short bearer token, and a JWT * after a hyphen. Each starts only where a run starts, or reads a bounded stretch from each start: linear. */ const BROAD: [RegExp, (mask: string) => string, string][] = [ [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], + // The root updater's own token rule on main, as it was: where it ends (a GitHub token's run of letters and digits, a + // Slack token's before a `_`) is where its next rule saw a boundary (maskUnion reads again there). + [/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, (k) => k, "the updater's token rule"], [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], - [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => `$1${k}`, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], - [/(^|[^a-z0-9+.-])([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, (k) => `$1$2${k}@`, 'user info in a URL'], + // A scheme of up to 32 characters right before `://`, from any start (after `-`, `.`, `+` or a letter too: the + // updater's own rule started at any `\b`, a scheme of any length): each start reads at most 33 characters, and each + // `://` is reached from at most 32 starts, so linear. + [/([a-z][a-z0-9+.-]{0,31}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], ]; @@ -200,9 +224,12 @@ export const REDACTION_PATTERNS: readonly { kind: string; re: RegExp; rep: Repla export interface RedactTextOptions { /** Also redact emails, internal hostnames, private IPs and home paths' user names (text for an issue, comment or PR). */ identifiers?: boolean; - /** Also apply the broad rules (BROAD), first: a token prefix cut short, a URL's user info, a value after a secret's name. */ + /** + * The broad mode: the shapes and the broad rules (BROAD: a token prefix cut short, a URL's user info, a value after a + * secret's name), all on the text as it was; the union of what they find is masked with `mask` (maskUnion). + */ broad?: boolean; - /** What a broad rule puts in place of what it finds; [REDACTED] when unset. */ + /** In the broad mode, what each masked run becomes (taken literally); [REDACTED] when unset. */ mask?: string; /** Keep the text's line count: a match over several lines (a private key) leaves its line breaks (redactRecords). */ keepLines?: boolean; @@ -210,23 +237,56 @@ export interface RedactTextOptions { /** * The prefixes of the shapes whose patterns start at `\b` (or after a character that isn't `_`): after an underscore - * (`MY_ghp_...`, `x_sk-...`) a second pass reads from the prefix on (afterUnderscore), so a key glued to a name is - * found too (#889). The first pass is unchanged, so nothing it found is lost. + * (`MY_ghp_...`, `x_sk-...`) a token is glued to a name, and the second pass (glued) reads it from its prefix on. */ -const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', DASHBOARD_KEY_PREFIX]; +const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', DASHBOARD_KEY_PREFIX]; +/** + * What ends in a letter and may sit right before a token (#889): an ANSI escape sequence (a coloured log line, + * `ESC[1m`) and a backslash escape in raw JSON or C-style text (`\n`, `\t`, `\"`, `\u001b`). A cut point goes after + * each. Each ESC or backslash starts one bounded match: linear. + */ +const GLUE = /\u001b(?:\[[0-?]*[ -/]*[@-~]|[@-Z\\-_])|\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])/g; -/** `s` with `f` applied again from each underscore that a known prefix follows: indexOf scans, so linear. */ -function afterUnderscore(s: string, f: (part: string) => string): string { - const cuts: number[] = []; +/** Where a token may start glued to what is before it: after each GLUE match, and after `_` before a known prefix. */ +function gluePoints(s: string): number[] { + const cuts = [...s.matchAll(GLUE)].map((m) => m.index! + m[0].length); for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) if (GLUED_PREFIXES.some((p) => s.startsWith(p, at + 1))) cuts.push(at + 1); + return [...new Set(cuts)].filter((c) => c < s.length).sort((a, b) => a - b); +} + +/** A character `s` doesn't hold (a private-use one), to mark the cut points with; null when it holds them all. */ +function freeMark(s: string): string | null { + const held = new Set(); + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xe000 && c <= 0xf8ff) held.add(c); + } + for (let c = 0xe000; c <= 0xf8ff; c++) if (!held.has(c)) return String.fromCharCode(c); + return null; +} + +/** + * The second pass (#889): `f` again on `s` with a boundary at each glue point, so a token glued to a name, to an ANSI + * escape or to a JSON escape is found from its start. The boundary is a mark no pattern takes in, inserted and taken + * out again, so the text is otherwise kept as it was. When `s` holds every mark character, each piece after a glue + * point is redacted on its own instead. The first pass ran on the text as it was, so nothing it found is lost. + */ +function glued(s: string, f: (part: string) => string): string { + const cuts = gluePoints(s); if (!cuts.length) return s; - let out = ''; + const mark = freeMark(s); + if (mark === null) { + let out = s.slice(0, cuts[0]); + for (const [i, c] of cuts.entries()) out += f(s.slice(c, cuts[i + 1] ?? s.length)); + return out; + } + let marked = ''; let from = 0; - for (const c of [...cuts, s.length]) { - out += from === 0 ? s.slice(0, c) : f(s.slice(from, c)); + for (const c of cuts) { + marked += s.slice(from, c) + mark; from = c; } - return out; + return f(marked + s.slice(from)).split(mark).join(''); } /** `$1`, `$&` and `$$` in a replacement string, as String.prototype.replace reads them. */ @@ -297,18 +357,154 @@ export function secretKinds(s: string): string[] { return [...kinds].sort(); } +/** A rule's regex as a global one, for matchAll (the orphan-END rule has no `g`). */ +const globals = new Map(); +const globalOf = (re: RegExp) => { + let g = globals.get(re); + if (!g) globals.set(re, (g = re.flags.includes('g') ? re : new RegExp(re.source, `${re.flags}g`))); + return g; +}; + +/** + * Where `re` finds a secret in `t`, as [start, end) spans: each match's part its replacement takes out (what the match + * and the replacement don't share at either end), so the scheme, the key's name or the separator a rule keeps is not + * in the span. A match its replacement leaves as it was (a function that judged it no secret) gives none. + */ +function spansOf(t: string, re: RegExp, rep: Replacement, out: [number, number][]) { + const g = globalOf(re); + g.lastIndex = 0; + for (let m = g.exec(t); m; m = g.exec(t)) { + const whole = m[0]; + // The next search starts 64 characters before the match's end, not after it: a value a rule takes in may hold the + // next secret's name (`token='abc password = 'x`, `Bearer abc+Bearer xyz`), and that next match must still be + // found. Each match start is after the last one's, and at most 64 characters are read again: linear. + g.lastIndex = Math.max(m.index + 1, m.index + whole.length - 64); + const groups = m.slice(1).map((x) => x ?? ''); + const r = typeof rep === 'string' ? expand(rep, whole, groups) : rep(whole, ...groups); + if (r === whole) continue; + let a = 0; + while (a < whole.length && a < r.length && whole[a] === r[a]) a++; + let b = 0; + while (b < whole.length - a && b < r.length - a && whole[whole.length - 1 - b] === r[r.length - 1 - b]) b++; + if (whole.length - b > a) out.push([m.index! + a, m.index! + whole.length - b]); + } +} + +/** Every rule of the broad mode: the shapes and the broad rules, each shown with the default mask. */ +const BROAD_RULES: [RegExp, Replacement][] = [...PATTERNS.map(([re, rep]): [RegExp, Replacement] => [re, rep]), ...BROAD.map(([re, rep]): [RegExp, Replacement] => [re, rep('[REDACTED]')])]; + +/** + * The broad mode (#889 round 3, the final check's regression finding): every rule runs on the text as it was, and the + * union of the spans they find is masked, each run once, with `mask`. Order-free by design: no rule's marker can hide + * what another rule would have found, so each rule masks at least what it masked alone (the modules' own rules on + * main among them; test/redact-broad-union.test.ts compares them on generated input). The glue points (gluePoints) + * are found the same way, on the text with a mark at each, mapped back. One pass of maskUnion. + */ +function maskOnce(s: string, mask: string, keepLines: boolean): string { + const spans: [number, number][] = []; + for (const [re, rep] of BROAD_RULES) spansOf(s, re, rep, spans); + // Two more reads, each with boundaries marked in: the glue points; and where a span found begins or ends between two + // word characters (a module's own rules, applied in turn, saw a boundary where an earlier rule's replacement ended: + // `xoxa-...[here]_db_password=...`). Apart, so a boundary of one kind can't break a match the other needs. + const word = (i: number) => /[A-Za-z0-9_]/.test(s[i] ?? ''); + readAt(s, gluePoints(s), spans); + readAt(s, [...new Set(spans.flat())].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), spans); + // What is already a mask stays as it is: a rule that takes in a mask word (`token=[REDACTED]`) adds nothing. + const masks: [number, number][] = []; + if (mask) for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) masks.push([at, at + mask.length]); + const kept = masks.length ? spans.flatMap(([a, b]) => outside([a, b], masks)) : spans; + if (!kept.length) return s; + kept.sort((x, y) => x[0] - y[0]); + let out = ''; + let at = 0; + let open: [number, number] | null = null; + const close = () => { + if (!open) return; + out += s.slice(at, open[0]) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(open[0], open[1]))) : ''); + at = open[1]; + }; + for (const [a, b] of kept) { + if (b <= a) continue; + if (open && a <= open[1]) open[1] = Math.max(open[1], b); + else { + close(); + open = [a, b]; + } + } + close(); + return out + s.slice(at); +} + +/** Every rule on `s` with a boundary marked in at each of `cuts` (sorted); the spans found, mapped back, go in `spans`. */ +function readAt(s: string, cuts: number[], spans: [number, number][]) { + if (!cuts.length) return; + const markChar = freeMark(s); + if (markChar === null) { + // Every mark character is in the text: each piece after a cut is read on its own. + for (const [i, c] of cuts.entries()) { + const piece = s.slice(c, cuts[i + 1] ?? s.length); + const more: [number, number][] = []; + for (const [re, rep] of BROAD_RULES) spansOf(piece, re, rep, more); + for (const [a, b] of more) spans.push([a + c, b + c]); + } + return; + } + let marked = ''; + let from = 0; + for (const c of cuts) { + marked += s.slice(from, c) + markChar; + from = c; + } + marked += s.slice(from); + // A position in the marked text, back to the text: less the marks before it. + const back = (p: number) => { + let lo = 0; + let hi = cuts.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (cuts[mid]! + mid < p) lo = mid + 1; + else hi = mid; + } + return p - lo; + }; + const more: [number, number][] = []; + for (const [re, rep] of BROAD_RULES) spansOf(marked, re, rep, more); + for (const [a, b] of more) spans.push([back(a), back(b)]); +} + +/** The parts of [a, b) outside every one of `regions` (sorted, not overlapping). */ +function outside([a, b]: [number, number], regions: [number, number][]): [number, number][] { + const out: [number, number][] = []; + let from = a; + for (const [x, y] of regions) { + if (y <= from || x >= b) continue; + if (x > from) out.push([from, x]); + from = Math.max(from, y); + } + if (from < b) out.push([from, b]); + return out; +} + +/** + * The broad mode: masked once, then again on what that gave, until nothing changes (at most 3 times). A mask can make + * a boundary a later rule needs (`[REDACTED]_db_password=...`), as each module's own rules, applied in turn, did. + */ +function maskUnion(s: string, mask: string, keepLines: boolean): string { + let out = maskOnce(s, mask, keepLines); + for (let i = 0; i < 2; i++) { + const again = maskOnce(out, mask, keepLines); + if (again === out) break; + out = again; + } + return out; +} + export function redactString(s: string, o: RedactTextOptions = {}): string { const put = o.keepLines ? applyKeepingLines : apply; - let out = s; - if (o.broad) { - // A private key first: a broad rule (a value after `TOKEN=`) would take its BEGIN line and leave its body unfound. - for (const [re, rep, kind] of PATTERNS) if (kind === 'private key') out = put(out, re, rep); - // A `$` in the mask is taken literally. - const mask = (o.mask ?? '[REDACTED]').replace(/\$/g, '$$$$'); - for (const [re, rep] of BROAD) out = put(out, re, rep(mask)); - } - const secrets = (t: string) => PATTERNS.reduce((x, [re, rep]) => put(x, re, rep), t); - out = afterUnderscore(secrets(out), secrets); + // The broad mode masks the union of every rule's spans (maskUnion); the default mode applies the shapes in turn, + // each with its own marker, then again from each glue point (glued). + const secrets = (t: string) => PATTERNS.reduce((y, [re, rep]) => put(y, re, rep), t); + let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : glued(secrets(s), secrets); if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); return out; } diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index e961512e9..09bb34f79 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -92,6 +92,7 @@ export const CHECK_TABLES: readonly CheckTable[] = [ // The root updater's generated copy of that module (test/redact-vendored.test.ts holds it to the module). { path: `scripts/machine/${BRAND.cli}-update.cjs`, table: 'PATTERNS', kind: 'never-shrink' }, { path: `scripts/machine/${BRAND.cli}-update.cjs`, table: 'CREDENTIAL_WORDS', kind: 'never-shrink' }, + { path: `scripts/machine/${BRAND.cli}-update.cjs`, table: 'BROAD_RULES', kind: 'never-shrink' }, { path: 'test/redact-records-keys.test.ts', table: 'CUT_SHORT', kind: 'never-shrink' }, { path: 'test/token-standins.ts', table: 'SHAPES', kind: 'never-shrink' }, { path: 'test/redaction-one-module.test.ts', table: 'BAD_SOURCES', kind: 'never-shrink' }, diff --git a/test/redact-broad-union.test.ts b/test/redact-broad-union.test.ts new file mode 100644 index 000000000..f85eb4f7c --- /dev/null +++ b/test/redact-broad-union.test.ts @@ -0,0 +1,180 @@ +// #889 round 3 (the round-2 final check's regression finding): the broad mode hides at least what each module's own +// rules hid on main. The root updater's redact(), gh-login's safeLine and the bench's scrub had their own token rules +// (kept here verbatim, as fixtures, from origin/main 41f6dd42); each now goes through the shared redactor's broad mode, +// which masks the union of what every rule finds on the text as it was. On generated input, every secret one of the +// old rules hid stays hidden, and so do the three classes the check found (user info after `-`, `.` or `+`; a bearer +// value that starts JWT-shaped and goes on; one rule's marker eating the next rule's text). +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { redactString } from '../src/events/redact.js'; +import { safeLine } from '../src/gh-login.js'; +import { scrub as benchScrub } from '../scripts/bench/cli.js'; +import { BRAND } from '../src/brand.js'; +import { repoRoot } from './helpers.js'; + +const updater = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`)) as { redact(text: string): string }; + +/** The root updater's redact() on main. */ +const oldUpdater = (text: string) => + String(text) + .replace(/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[redacted key]') + .replace(/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, '[redacted]') + .replace(/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 [redacted]') + .replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, '$1[redacted]@') + .replace(/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, '$1$2[redacted]'); +/** gh-login's TOKENISH on main. */ +const oldGhLogin = (line: string) => line.replace(/\b(gh[opusr]_[A-Za-z0-9_]{8,}|github_pat_[A-Za-z0-9_]{8,})/g, '[hidden]'); +/** The bench's token and email rules on main, then its control characters. */ +const oldBench = (text: string) => + text + .replace(/sk-ant-[\w-]+|\bgh[opsur]_\w{16,}|\bgithub_pat_\w{16,}|\bxox[abprs]-[\w-]+|\bAKIA[0-9A-Z]{16}\b|\beyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]+/g, '[redacted]') + .replace(/[\w.+-]+@[\w-]+(?:\.[\w-]+)*\.[A-Za-z]{2,}\b/g, '[email]') + .replace(/[\p{Cc}\p{Cf}]/gu, ' '); + +/** A small seeded generator, so a failure names an input that can be run again. */ +function rng(seed: number) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const ESC = '\u001b'; +// A key's BEGIN and END lines, put together at run time so no key-shaped literal is committed. +const KEY_BEGIN = ['-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const KEY_END = ['-----END', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const ALNUM = 'ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789'; + +/** One generated input and the secrets in it: the random parts a rule exists to hide. */ +function generate(r: () => number): { text: string; secrets: string[] } { + const pick = (xs: readonly T[]) => xs[Math.floor(r() * xs.length)]!; + const word = (n: number, chars = ALNUM) => Array.from({ length: n }, () => pick([...chars])).join(''); + const secrets: string[] = []; + const secret = (n: number, chars = ALNUM) => { + const s = word(n, chars); + secrets.push(s); + return s; + }; + const jwt = () => `eyJ${secret(12)}.${secret(14)}.${secret(16)}`; + const segments: (() => string)[] = [ + () => `${pick(['ghp', 'gho', 'ghu', 'ghs', 'ghr'])}_${secret(8 + Math.floor(r() * 32))}`, + () => `github_pat_${secret(10)}_${secret(10 + Math.floor(r() * 50))}`, + () => `sk-ant-${pick(['', 'api03-', 'oat01-'])}${secret(8 + Math.floor(r() * 40))}`, + () => `${pick(['xoxb', 'xoxp', 'xoxa', 'xoxr', 'xoxs'])}-${secret(4 + Math.floor(r() * 30))}`, + () => `${pick(['Bearer', 'bearer', 'BEARER'])} ${secret(8 + Math.floor(r() * 30), `${ALNUM}._~+/-`)}`, + () => `${pick(['Bearer', 'bearer'])} ${jwt()}`, + () => `Bearer ${jwt()}.${secret(20)}.${secret(16)}`, + () => `Bearer ${jwt()}${pick(['+/', '==', '.x'])}${secret(12)}`, + () => `${pick(['https', 'git+https', 'ssh', 'http'])}://${pick(['', `${word(5)}:`])}${secret(10 + Math.floor(r() * 20))}@${pick(['github.com', 'git.example.test'])}/o/r.git`, + () => `${pick(['token', 'GH_TOKEN', 'api_key', 'apikey', 'secret', 'password', 'db_password', 'x-api-key', 'passwd'])}${pick(['=', ': ', ':', ' = '])}${pick(['', '"', "'"])}${secret(8 + Math.floor(r() * 20))}`, + () => jwt(), + () => `${KEY_BEGIN}\n${secret(64)}\n${KEY_END}`, + ]; + const joiners = ['', ' ', '-', '.', '+', '_', '/', ':', '=', '"', ', ', '\n', `${ESC}[1m`, '\\n', ' -', ' +', ' .']; + const heads = ['', 'error: ', 'x', 'apikey', 'see ', 'clone failed: ']; + let text = pick(heads); + const n = 1 + Math.floor(r() * 4); + for (let i = 0; i < n; i++) text += (i ? pick(joiners) : '') + pick(segments)(); + return { text: text + pick(['', ' tail', '\n', '"']), secrets }; +} + +/** How many inputs per module, and a seed offset: fixed in CI; a longer run sets REDACT_FUZZ_N and REDACT_FUZZ_SEED. */ +const N = Number(process.env.REDACT_FUZZ_N) || 6000; +const SEED = Number(process.env.REDACT_FUZZ_SEED) || 0; + +/** The inputs where the new output shows a secret the old one hid, at most `max` of them. */ +function narrowings(old: (t: string) => string, now: (t: string) => string, seed: number, n: number, max = 10): string[] { + const r = rng(seed); + const found: string[] = []; + for (let i = 0; i < n && found.length < max; i++) { + const { text, secrets } = generate(r); + const a = old(text); + const b = now(text); + for (const s of secrets) if (!a.includes(s) && b.includes(s)) found.push(`${JSON.stringify(text)} -> ${JSON.stringify(b)}`); + } + return found; +} + +test("the root updater: on generated input, every secret main's five rules hid is still hidden", () => { + assert.deepEqual(narrowings(oldUpdater, (t) => updater.redact(t), SEED + 889, N), []); +}); + +test("gh-login: on generated input, every secret main's rule hid is still hidden", () => { + assert.deepEqual(narrowings(oldGhLogin, (t) => redactString(t, { broad: true, mask: '[hidden]' } as Parameters[1]), SEED + 921, N), []); + // safeLine itself: one line, cut to 200 characters. + const r = rng(SEED + 7); + const found: string[] = []; + for (let i = 0; i < N / 2; i++) { + const { text, secrets } = generate(r); + const line = text.replace(/\n/g, ' '); + // Hidden by the old rule (not only cut off by the 200-character cut), and shown now. + for (const s of secrets) if (!oldGhLogin(line).includes(s) && safeLine(line).includes(s)) found.push(line); + } + assert.deepEqual(found.slice(0, 10), []); +}); + +test("the bench: on generated input, every secret main's rules hid is still hidden", () => { + assert.deepEqual(narrowings(oldBench, (t) => benchScrub(t, {}, 1_000_000), SEED + 1729, N), []); +}); + +/** The final check's fragment fuzz: secrets and pieces of token shapes, joined at random; none shown more often now. */ +const SECRETS = ['Q1w2E3r4', 'Z9y8X7w6V5u4', 'Mn3Bv4Cx5Zl6Kj7H', 'pL0oK9iJ8uH7yG6tF5rD4eS3']; +const REAL = [`${'ghp'}_${SECRETS[3]}Mn3Bv4Cx5Zl6`, `github_pat_11${SECRETS[3]}_Mn3Bv4Cx5Zl6Kj7H`, `${'sk'}-ant-api03-${SECRETS[3]}`, `${'xoxb'}-123456789012-${SECRETS[3]}`, `${'eyJ'}hbGciOiJIUzI1NiJ9.${'eyJ'}zdWIiOiIxIn0.${SECRETS[3]}`, `${'AKIA'}Q7RZ4KX2M9TBQ7RZ`, `tskey-auth-k${SECRETS[3]}`, `wld_${SECRETS[3]}Mn3Bv4Cx5Z`]; +const PIECES: readonly (readonly string[])[] = [ + [...REAL, '.', 'bearer ', 'Bearer ', 'https://', 'git+https://', '@', ':', '=', ' ', ' ', '-', '+', '_', 'token', 'TOKEN', 'Password', 'secret', 'api_key', '"', "'", '/', '\n', `${KEY_BEGIN}\n`, `\n${KEY_END}`, 'x', ',', 'Authorization: ', 'fatal: ', `${ESC}[1m`, '\\n'], + ['ghp_', 'gho_', 'github_pat_', 'sk-ant-', 'xoxb-', 'xoxp-', 'eyJ', '.', 'bearer ', 'Bearer ', 'BEARER ', 'https://', 'git+ssh://', '@', ':', '=', ' ', ' ', '-', '+', '_', 'token', 'TOKEN', 'Password', 'secret', 'api_key', 'apikey', '"', "'", '/', '\n', KEY_BEGIN, KEY_END, 'AKIA', 'wld_', 'tskey-', '0x', 'x', ',', `${ESC}[1m`, '\\n', 'Basic '], +]; + +test("the final check's fragment fuzz: no module shows a secret more often than its own rules on main did", () => { + const r = rng(SEED + 2463); + const pick = (xs: readonly T[]) => xs[Math.floor(r() * xs.length)]!; + const mods: [string, (t: string) => string, (t: string) => string][] = [ + ['updater', oldUpdater, (t) => updater.redact(t)], + ['gh-login', oldGhLogin, (t) => redactString(t, { broad: true, mask: '[hidden]' } as Parameters[1])], + ['bench', oldBench, (t) => benchScrub(t, {}, 1_000_000)], + ]; + const found: string[] = []; + for (let i = 0; i < N * 2 && found.length < 10; i++) { + const pieces = pick(PIECES); + let s = ''; + for (let j = 0, k = 2 + Math.floor(r() * 6); j < k; j++) s += r() < 1 / 3 ? pick(SECRETS) : pick(pieces); + for (const [who, old, now] of mods) { + const a = old(s); + const b = now(s); + for (const x of SECRETS) if (b.split(x).length > a.split(x).length) found.push(`${who}: ${JSON.stringify(s)} -> ${JSON.stringify(b)}`); + } + } + assert.deepEqual(found, []); +}); + +test('the three classes the final check found, each hidden by the updater, gh-login and the bench', () => { + const tok = `${'ghp'}_${'Zq9Xw8Vu7Ts6Rq5Po4Nm3Lk2Ji1Hg0FeDcBa'}`; + const jwt = `eyJ${'hbGciOiJIUzI1NiJ9'}.eyJ${'zdWIiOiJmYWtlIn0'}.${'c2lnbmF0dXJlMTIz'}`; + const tail = 'Q2lwaGVydGV4dEFuZFRhZw'; + const s = 'PrivSecretValue9876'; + const cases: [string, string][] = [ + [`error: -https://${tok}@github.com/o/r.git`, tok], + [`x +git+https://${tok}@github.com/o/r.git`, tok], + [`see .https://${tok}@github.com`, tok], + [`Authorization: Bearer ${jwt}.${tail}.${tail}`, tail], + [`Bearer ${jwt}.${tail}`, tail], + [`Bearer ${jwt}+/${tail}==`, tail], + [`xoxb-Bearer ${s}`, s], + [`apikeysk-ant-secret:${s}`, s], + [`sk-ant-BEARER ${s}`, s], + ]; + const missed: string[] = []; + for (const [text, secret] of cases) { + for (const [who, out] of [['updater', updater.redact(text)], ['broad', redactString(text, { broad: true } as Parameters[1])], ['bench', benchScrub(text, {})]] as const) { + if (out.includes(secret)) missed.push(`${who}: ${out}`); + } + } + assert.deepEqual(missed, []); +}); diff --git a/test/redact-fail-closed.test.ts b/test/redact-fail-closed.test.ts new file mode 100644 index 000000000..819d19022 --- /dev/null +++ b/test/redact-fail-closed.test.ts @@ -0,0 +1,163 @@ +// #889 round 3 (the second adversarial review): the redactor fails closed on what it found glued or cut. +// - A token right after an ANSI escape (a coloured log line) or a JSON escape (`\n` in raw JSON) is found, on every path. +// - A private key with no END is redacted to the end of the text, one with no BEGIN from the start; PGP's key block too. +// - HTTP Basic credentials are redacted; a dashboard key after `-` or a letter, and `ghp_..._suffix`, are found. +// - The sites that cut text redact it first; doctor checks the hub's dashboard key too. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as redactModule from '../src/events/redact.js'; +import { redactString } from '../src/events/redact.js'; +import { readServiceLog } from '../src/dashboard.js'; +import { safeLine } from '../src/gh-login.js'; +import { scrub as benchScrub } from '../scripts/bench/cli.js'; +import { prSections } from '../src/decision-detail.js'; +import { attachmentPrompt, ATTACHMENT_DEFAULT_CAPS, storeUpload, INLINE_PER_FILE } from '../src/chat-attachments.js'; +import * as doctorModule from '../src/doctor.js'; +import { stateDir } from '../src/os/index.js'; +import { BRAND } from '../src/brand.js'; +import { exampleProject, repoRoot } from './helpers.js'; +import { leaks, run, SHAPES } from './token-standins.js'; + +const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); +const updater = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`)) as { redact(text: string): string }; +const records = (t: string) => (typeof redactModule.redactRecords === 'function' ? redactModule.redactRecords(t.split('\n')).join('\n') : 'no redactRecords'); +const journalView = (msgs: string[]) => { + const journal = () => ({ status: 0, stderr: '', stdout: msgs.map((m, i) => JSON.stringify({ MESSAGE: m, PRIORITY: '6', __REALTIME_TIMESTAMP: String(1_767_225_600_000_000 + i) })).join('\n') }); + return JSON.stringify(readServiceLog({ unit: 'coordinator.service', file: null }, 200, journal).entries); +}; +const fileView = (text: string) => { + const file = join(scratch('svc-log-'), 'c.log'); + writeFileSync(file, `${text}\n`); + return JSON.stringify(readServiceLog({ unit: null, file }, 200).entries); +}; +/** Every path a line can take: the shared redactor's modes, the dashboard's views and the three modules that use it. */ +const PATHS: [string, (t: string) => string][] = [ + ['redactString', (t) => redactString(t)], + ['redactString broad', (t) => redactString(t, { broad: true } as Parameters[1])], + ['redactRecords', records], + ['journal view', (t) => journalView(t.split('\n'))], + ['file view', fileView], + ['gh-login', (t) => t.split('\n').map((l) => safeLine(l)).join('\n')], + ['bench', (t) => benchScrub(t, {}, 100_000)], + ['updater', (t) => updater.redact(t)], +]; +const ESC = '\u001b'; +const BS = '\\'; + +test('a token right after an ANSI colour code or a JSON escape is found, every token shape on every path', () => { + const missed: string[] = []; + const wraps: [string, (v: string) => string][] = [ + ['ANSI bold', (v) => `${ESC}[1m${v}${ESC}[0m`], + ['ANSI colour after text', (v) => `error: ${ESC}[31;1m${v}${ESC}[0m failed`], + ['JSON \\n', (v) => `{"log":"first line${BS}n${v}"}`], + ['JSON \\t', (v) => `"a${BS}t${v}"`], + ['JSON \\"', (v) => `"say ${BS}"${v}${BS}""`], + ]; + for (const s of SHAPES.filter((x) => x.token && !x.value.includes('\n'))) { + for (const [how, wrap] of wraps) { + for (const [path, out] of PATHS) { + if (path === 'gh-login' && !s.kind.startsWith('GitHub')) continue; + if (leaks(out(wrap(s.value)), s.secret)) missed.push(`${s.what}, ${how}, via ${path}`); + } + } + } + assert.deepEqual(missed, []); + // Ordinary coloured lines and JSON text are left as they were. + for (const s of [`${ESC}[32mok${ESC}[0m: 3 passed`, `{"msg":"line one${BS}nline two${BS}tdone"}`, 'C:\\Users\\example\\notes.txt']) assert.equal(redactString(s), s); +}); + +/** A 4096-bit-sized key: 50 body lines between its BEGIN and END lines. */ +const BODY = Array.from({ length: 50 }, (_, i) => run(64, `K${i}eyBody${i}x9`)); +const begin = (t = 'RSA') => `-----BEGIN ${t} PRIVATE KEY-----`; +const end = (t = 'RSA') => `-----END ${t} PRIVATE KEY-----`; +const bodyLeaks = (out: string) => BODY.filter((b) => leaks(out, b)).length; + +test('a private key with no END line (cut short, or still being written) is redacted to the end, on every path', () => { + const missed: string[] = []; + const text = ['starting', begin(), ...BODY.slice(0, 20)].join('\n'); + for (const [path, out] of PATHS) if (path !== 'gh-login' && bodyLeaks(out(text))) missed.push(`${path}: ${bodyLeaks(out(text))} body lines shown`); + assert.deepEqual(missed, []); + assert.match(redactString(text), /^starting\n\[REDACTED PRIVATE KEY\]$/); +}); + +test('a private key whose BEGIN is above the window (an END with no BEGIN) is redacted from the start, on every path', () => { + const missed: string[] = []; + // The updater's journal tail: `journalctl -n 30` of a service that logged a 50-line key; and the dashboard's views. + const tail = [...BODY.slice(-28), end(), 'service started'].join('\n'); + for (const [path, out] of PATHS) if (path !== 'gh-login' && bodyLeaks(out(tail))) missed.push(`${path}: ${bodyLeaks(out(tail))} body lines shown`); + assert.deepEqual(missed, []); + assert.match(redactString(tail), /^\[REDACTED PRIVATE KEY\]\nservice started$/); +}); + +test("PGP's private key block, and an OpenSSH or encrypted PEM key, are redacted", () => { + for (const type of ['PGP', 'OPENSSH', 'ENCRYPTED']) { + const pgp = type === 'PGP'; + const key = [`-----BEGIN ${type} PRIVATE KEY${pgp ? ' BLOCK' : ''}-----`, ...BODY.slice(0, 5), `-----END ${type} PRIVATE KEY${pgp ? ' BLOCK' : ''}-----`].join('\n'); + for (const [path, out] of PATHS) if (path !== 'gh-login') assert.equal(bodyLeaks(out(`key: ${key} done`)), 0, `${type} via ${path}`); + } +}); + +test('HTTP Basic credentials are redacted, plain and as a JSON header; prose after "Basic" is not', () => { + const cred = Buffer.from(`x-access-token:${'ghs'}_${run(36)}`).toString('base64'); + const missed: string[] = []; + for (const line of [`Authorization: Basic ${cred}`, `{"headers":{"Authorization":"Basic ${cred}"}}`, `git -c http.extraheader="AUTHORIZATION: basic ${cred}" fetch`]) { + for (const [path, out] of PATHS) if (path !== 'gh-login' && leaks(out(line), cred)) missed.push(`${line.slice(0, 20)}… via ${path}`); + } + assert.deepEqual(missed, []); + for (const s of ['Basic Authentication is off', 'basic configuration only', 'BASIC usage']) assert.equal(redactString(s), s); +}); + +test('a dashboard key after a hyphen or a letter, and a GitHub token followed by `_`, are found by default', () => { + const key = `wld_${run(32, 'Dk7-qP_2')}`; + for (const s of [`dashboard-key-${key}`, `token${key}`, `key=${key}`]) assert.ok(!leaks(redactString(s), key), s); + const gh = `${'ghp'}_${run(36)}`; + assert.ok(!leaks(redactString(`${gh}_backup`), gh), 'ghp_..._backup'); + for (const s of [`xox${'b'}-${run(24)}_x`, `${'lin'}_api_${run(40)}_x`, `AKIA${'Q7RZ4KX2M9TBQ7RZ'}_x`]) assert.ok(!leaks(redactString(s), s.slice(0, -2)), s); +}); + +test('text cut to a size is redacted before the cut: a PR section, and a chat attachment', async () => { + const gh = `${'ghp'}_${run(36)}`; + // A token across the PR section's 20,000-character cut: its first 20 characters inside. + const body = `## Fail-without\n${'x'.repeat(20_000 - 21)} ${gh} tail\n\n## Next\n`; + const section = prSections(body).failWithout!; + assert.ok(!section.includes(gh.slice(0, 20)), 'no piece of the token is kept'); + // An attachment cut at INLINE_PER_FILE, a token across the cut. + const s = scratch('attach-'); + async function* once(b: string) { + yield Buffer.from(b); + } + const m = await storeUpload({ stateDir: s, name: 'log.txt', body: once(`${'y'.repeat(INLINE_PER_FILE - 21)} ${gh} tail`), caps: { ...ATTACHMENT_DEFAULT_CAPS }, by: 'example-collaborator', gitleaks: null, anyway: true }); + const prompt = attachmentPrompt(s, [m.id], scratch('bundle-')).text; + assert.ok(!prompt.includes(gh.slice(0, 20)), 'no piece of the token is kept'); +}); + +test("doctor checks the hub's dashboard key too", () => { + const hub = join(stateDir(), 'dashboard-hub'); + mkdirSync(hub, { recursive: true }); + const old = run(32, 'Hb9OldKey_-'); + writeFileSync(join(hub, 'dashboard-token'), old, { mode: 0o600 }); + const { dir: root } = exampleProject(); + const all = doctorModule.doctor(root); + assert.equal(all.find((c) => c.name === 'dashboard key hub')?.level, 'warn'); + assert.ok(!JSON.stringify(all).includes(old), 'the key is never shown'); +}); + +test('the glue pass and the fail-closed key rules run in linear time on hostile input', () => { + const N = 100_000; + const fill = (unit: string) => unit.repeat(Math.ceil(N / unit.length)); + const slow: string[] = []; + for (const s of [fill(`${ESC}[1m`), fill(`${BS}n`), fill(`${BS}u12`), fill(`${ESC}[`), fill(`${ESC}[1;`), fill(begin()), fill(end()), fill('-----BEGIN '), fill('-----END '), fill('Basic '), fill('Basic aaaaaaaa'), fill('wld_'), fill(`${BS}nghp_`)]) { + for (const [what, f] of [['default', (t: string) => redactString(t)], ['broad', (t: string) => redactString(t, { broad: true } as Parameters[1])]] as const) { + const t0 = performance.now(); + f(s); + const ms = performance.now() - t0; + if (ms > 2000) slow.push(`${what} on ${JSON.stringify(s.slice(0, 10))}…: ${Math.round(ms)} ms`); + } + } + assert.deepEqual(slow, []); +}); diff --git a/test/token-standins.ts b/test/token-standins.ts index e2915c84b..8bf3455e6 100644 --- a/test/token-standins.ts +++ b/test/token-standins.ts @@ -40,6 +40,8 @@ const PEM_BODY = run(64); const PEM = j('\n', j(' ', '-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'), PEM_BODY, j(' ', '-----END', 'RSA', 'PRIVATE', 'KEY-----')); const URL_PASSWORD = run(24, 'pW9xQ2zL'); const JSON_VALUE = run(24, 'notArealValue7'); +/** HTTP Basic: base64 of user:password (git's extraheader form, with an App installation token as the password). */ +const BASIC = Buffer.from(`x-access-token:${j('_', 'ghs', run(36))}`).toString('base64'); /** Every stand-in: a shape gone from here is a shape no test drives (registered never-shrink). */ export const SHAPES: readonly StandIn[] = [ @@ -86,6 +88,7 @@ export const SHAPES: readonly StandIn[] = [ t('AWS access key', 'an AWS temporary key id', `ASIA${upper(16)}`), t('private key', 'a PEM private key', PEM, PEM_BODY), t('bearer token', 'a bearer token', `Bearer ${run(40, 'zY8xW7vU6')}`, run(40, 'zY8xW7vU6')), + t('Basic credential', 'a Basic credential', `Basic ${BASIC}`, BASIC), t('32-byte hex key', 'a 32-byte hex key', `0x${hex(64)}`), // Structures around a value, not token shapes. { kind: 'password in a URL', what: 'a password in a URL', value: `postgres://app:${URL_PASSWORD}@db.example.test:5432/app`, secret: URL_PASSWORD, token: false }, From 88ec30b7cfbee831bfc732e577a4cc4130525d0a Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 00:57:02 -0500 Subject: [PATCH 5/9] redaction: distinctive prefixes match anywhere, every cut site windowed, linear URL rule (#889) Round 4, on the third adversarial review (#921): - Every shape with a distinctive prefix (gh*_, github_pat_, sk-ant-, xox?-, xapp-, lin_*, AKIA/ASIA, AIza, eyJ with a bounded header, wld_, tskey-) is matched with no left boundary, in default and broad mode, so nothing glued before it hides it. Shapes with no distinctive prefix keep a boundary, and the glue points now take ESC in every encoding (the byte, \u001b, \x1b, \033, \e, ^[), each sequence form (CSI, nF, single character) and the 8-bit CSI. - Bearer and Basic match in any case. - The default URL-password rule takes a scheme of up to 32 characters from any start: linear on long dotted runs (it was quadratic). redact.test's linear-time inputs gain dotted runs, and its KNOWN_SLOW exemption for that rule is emptied. - redactCut: decisionDiff, a PR section, a chat attachment and the dashboard's log views redact only what they keep plus a margin (16 KB; 4 KB per log record), not the whole input. - Broad mode's mask subtraction walks the sorted spans and masks once. Test (fails without): test/redact-escapes-windows.test.ts at d0f4ab94: 3 fail and 2 did not finish in 280 s (the linear-time and window tests); 1 passes (a key cut at a window edge). Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 6 +- scripts/machine/worklane-update.cjs | 101 +++++++++------- src/chat-attachments.ts | 7 +- src/dashboard.ts | 17 ++- src/decision-detail.ts | 12 +- src/events/redact.ts | 130 ++++++++++++++------- test/redact-escapes-windows.test.ts | 154 +++++++++++++++++++++++++ test/redact.test.ts | 6 +- 8 files changed, 328 insertions(+), 105 deletions(-) create mode 100644 test/redact-escapes-windows.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index 7ca636d16..31464517d 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -4,8 +4,8 @@ ### Guardrail changes: what each one does -- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. It also finds a token glued to a name after `_` (`MY_ghp_...`), right after an ANSI colour code or a JSON escape (`\n`), and after a lowercase `bearer`. Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. -- **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. PGP's private key block is covered too. A key whose BEGIN and END are both outside a window (only its middle lines in view) is not recognised. +- **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. A shape with a distinctive prefix (`ghp_`, `github_pat_`, `sk-ant-`, `xoxb-`, `lin_api_`, `AKIA`, `AIza`, `eyJ`, `wld_`, ...) is found wherever its prefix is, whatever is glued before it. The other shapes (a Discord bot token, a Bearer or Basic credential in any case, a Stripe key, a plain `sk-` key) are found after a name glued with `_`, an escape sequence (ESC as a byte, as `\u001b`, `\x1b`, `\033`, `\e` or `^[`, and the 8-bit CSI) or a JSON escape (`\n`). Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. +- **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. This covers PEM keys (upper-case BEGIN and END lines with five dashes) and PGP's private key block. Not covered: a key whose BEGIN and END are both outside a window (only its middle lines in view), and other formats (an ssh.com SSH2 key, a lower-case PEM line, a PuTTY `.ppk` file). - **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. -- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section and a chat attachment are redacted before they are cut, too. +- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. The URL-password rule no longer slows down on a long dotted line. - A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 4ae34b13f..41193ade0 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -115,9 +115,10 @@ const sharedRedact = (() => { const exports = {}; "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; +exports.CUT_MARGIN = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; exports.secretKinds = secretKinds; exports.redactString = redactString; +exports.redactCut = redactCut; exports.redactRecords = redactRecords; exports.redact = redact; const credentialShaped = (v) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); @@ -146,24 +147,26 @@ const isDashboardKey = (key) => new RegExp(`^${exports.DASHBOARD_KEY_PREFIX}${DA exports.isDashboardKey = isDashboardKey; const PATTERNS = [ [new RegExp(`${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], - [/\b(gh[pousr]_[A-Za-z0-9]{30,})(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], - [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], + [/gh[pousr]_[A-Za-z0-9]{30,}(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], + [/github_pat_[A-Za-z0-9_]{40,}/g, 'github_pat_[REDACTED]', 'GitHub token'], + [/sk-ant-[A-Za-z0-9_-]{20,}/g, 'sk-[REDACTED]', 'API key (sk-)'], [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], - [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]', 'Google API key'], + [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], - [/\b(AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], - [/\bxoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], - [/\bxox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], - [/\bxapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], - [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], + [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '[REDACTED JWT]', 'JWT'], + [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], + [/xoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], + [/xox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], + [/xapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], + [/lin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], - [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], - [/\b(Basic|basic|BASIC)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/g, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], - [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], + [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], + [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/gi, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], + [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], [ /(\\?")([A-Za-z0-9_-]+)\1(\s*:\s*)\1([A-Za-z0-9_\-.+/=~]{16,})\1/g, @@ -173,11 +176,11 @@ const PATTERNS = [ [/(^|[^A-Za-z0-9])[A-Za-z0-9_-]{23,28}\.[A-Za-z0-9_-]{6,7}\.[A-Za-z0-9_-]{27,}\b/g, '$1[REDACTED]', 'Discord bot token'], ]; const BROAD = [ - [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], + [/(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => k, 'GitHub token, cut short'], [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], [/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, (k) => k, "the updater's token rule"], - [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], - [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/xox[abprs]-[A-Za-z0-9_-]+/g, (k) => k, 'Slack token, cut short'], + [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => k, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], [/([a-z][a-z0-9+.-]{0,31}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], @@ -217,7 +220,11 @@ exports.REDACTION_PATTERNS = [ ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'identifier' })), ]; const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', exports.DASHBOARD_KEY_PREFIX]; -const GLUE = /\u001b(?:\[[0-?]*[ -/]*[@-~]|[@-Z\\-_])|\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])/g; +const GLUE = new RegExp([ + String.raw `(?:\u001b|\\u001[bB]|\\x1[bB]|\\033|\\e|\^\[)(?:\[[0-?]*[ -/]*[@-~]|[ -/]+[0-~]|[0-~])`, + String.raw `\u009b[0-?]*[ -/]*[@-~]`, + String.raw `\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])`, +].join('|'), 'g'); function gluePoints(s) { const cuts = [...s.matchAll(GLUE)].map((m) => m.index + m[0].length); for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) @@ -348,32 +355,29 @@ function maskOnce(s, mask, keepLines) { if (mask) for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) masks.push([at, at + mask.length]); - const kept = masks.length ? spans.flatMap(([a, b]) => outside([a, b], masks)) : spans; + const kept = outside(merged(spans), masks); if (!kept.length) return s; - kept.sort((x, y) => x[0] - y[0]); let out = ''; let at = 0; - let open = null; - const close = () => { - if (!open) - return; - out += s.slice(at, open[0]) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(open[0], open[1]))) : ''); - at = open[1]; - }; for (const [a, b] of kept) { - if (b <= a) - continue; - if (open && a <= open[1]) - open[1] = Math.max(open[1], b); - else { - close(); - open = [a, b]; - } + out += s.slice(at, a) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(a, b))) : ''); + at = b; } - close(); return out + s.slice(at); } +function merged(spans) { + const sorted = spans.filter(([a, b]) => b > a).sort((x, y) => x[0] - y[0]); + const out = []; + for (const [a, b] of sorted) { + const last = out.at(-1); + if (last && a <= last[1]) + last[1] = Math.max(last[1], b); + else + out.push([a, b]); + } + return out; +} function readAt(s, cuts, spans) { if (!cuts.length) return; @@ -414,18 +418,22 @@ function readAt(s, cuts, spans) { for (const [a, b] of more) spans.push([back(a), back(b)]); } -function outside([a, b], regions) { +function outside(spans, regions) { const out = []; - let from = a; - for (const [x, y] of regions) { - if (y <= from || x >= b) - continue; - if (x > from) - out.push([from, x]); - from = Math.max(from, y); + let r = 0; + for (const [a, b] of spans) { + while (r < regions.length && regions[r][1] <= a) + r++; + let from = a; + for (let k = r; k < regions.length && regions[k][0] < b; k++) { + const [x, y] = regions[k]; + if (x > from) + out.push([from, x]); + from = Math.max(from, y); + } + if (from < b) + out.push([from, b]); } - if (from < b) - out.push([from, b]); return out; } function maskUnion(s, mask, keepLines) { @@ -446,6 +454,11 @@ function redactString(s, o = {}) { out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); return out; } +exports.CUT_MARGIN = 16 * 1024; +function redactCut(s, max, o = {}) { + const r = redactString(s.length > max + exports.CUT_MARGIN ? s.slice(0, max + exports.CUT_MARGIN) : s, o); + return r.length > max ? r.slice(0, max) : r; +} function redactRecords(records, o = {}) { const lines = redactString(records.join('\n'), { ...o, keepLines: true }).split('\n'); const out = []; diff --git a/src/chat-attachments.ts b/src/chat-attachments.ts index f23365c9f..c9b734b71 100644 --- a/src/chat-attachments.ts +++ b/src/chat-attachments.ts @@ -16,7 +16,7 @@ import { createHash, randomBytes } from 'node:crypto'; import { closeSync, existsSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeSync } from 'node:fs'; import { join } from 'node:path'; import type { InstanceSettings } from './instance.js'; -import { redactString, secretKinds } from './events/redact.js'; +import { redactCut, redactString, secretKinds } from './events/redact.js'; import { ownerOnlyDir, writeOwnerOnly } from './os/index.js'; import { scanFile } from './scan/secrets.js'; @@ -370,8 +370,9 @@ export function attachmentPrompt(stateDir: string, ids: string[], bundleDir: str } else { const raw = attachmentBytes(stateDir, id)!.toString('utf8'); const room = Math.max(0, Math.min(INLINE_PER_FILE, budget)); - // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. - const body = redactString(raw).slice(0, room); + // Redacted before it is cut, over the kept part and a margin only (#889): a cut first leaves a token or a key's + // body unfound, and a file may be 10 MB. + const body = redactCut(raw, room); budget -= body.length; const cut = raw.length > room ? `\n(cut: the first ${room} of ${raw.length} characters)` : ''; lines.push(`${what}:`, `<<>>`, body, `<<>>${cut}`); diff --git a/src/dashboard.ts b/src/dashboard.ts index 1c5e8cf43..0e00cdecc 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -36,7 +36,7 @@ import { EventCache, type OpenRead } from './events/cache.js'; import { compress, compressibleType, MIN_COMPRESS_BYTES, mayCompressApi, requestEncoding, StaticCompressed, type Encoding } from './dashboard-compress.js'; import { dashboardProof, NONCE, PROOF_PATH } from './dashboard-proof.js'; import { localDay } from './local-day.js'; -import { DASHBOARD_KEY_PREFIX, redactRecords, redactString } from './events/redact.js'; +import { DASHBOARD_KEY_PREFIX, redactCut, redactRecords, redactString } from './events/redact.js'; import type { EventPayload, StoredEvent } from './events/types.js'; import { acceptGit, acceptMerge, protectionView, type AcceptGit, type AcceptTarget } from './main-watch.js'; import { deployGateView } from './deploy-gate/gate.js'; @@ -884,10 +884,16 @@ const journalctl: JournalReader = (unit, lines) => { return { status: r.status, stdout: r.stdout ?? '', stderr: `${r.stderr ?? ''}${r.error ? r.error.message : ''}` }; }; +/** A message cut to 2,000 characters, redacted first: a cut made before would leave a long secret's body unfound (#889). */ +/** How much of one log record is shown, and how much of it is read (redacted) to show that (#889 round 4). */ +const RECORD_SHOWN = 2000; +const RECORD_READ = RECORD_SHOWN + 4096; +/** A record cut to what is read of it: a huge journal line costs no more than that (the redaction is synchronous). */ +const readPart = (s: string) => (s.length > RECORD_READ ? s.slice(0, RECORD_READ) : s); /** A message cut to 2,000 characters, redacted first: a cut made before would leave a long secret's body unfound (#889). */ const clip = (s: string) => { - const r = redactString(s); - return r.length > 2000 ? `${r.slice(0, 2000)} …` : r; + const r = redactCut(s, RECORD_SHOWN); + return s.length > RECORD_SHOWN && r.length >= RECORD_SHOWN ? `${r} …` : r; }; /** * Records read above the kept ones (journal entries or log-file lines), so a private key that starts there is @@ -922,7 +928,8 @@ export function readServiceLog(src: LogSource, lines = 200, read: JournalReader } // journald keeps a service's output one line per entry: the entries are redacted as one text (#889), so a private // key logged a line at a time is found whole. An entry that was inside one comes back empty and is left out. - const shown = redactRecords(records.map((e) => e.message)); + // Each entry only as far as it is read (RECORD_READ), so a run of huge entries costs no more than the view shows. + const shown = redactRecords(records.map((e) => readPart(e.message))); const entries = records.flatMap((e, i) => (shown[i]!.trim() || !e.message.trim() ? [{ ...e, message: clip(shown[i]!) }] : [])).slice(-lines); if (entries.length) return { source: 'journal', unit: src.unit, entries, problem: null }; problem = `no lines of ${src.unit} this user can read. Either the service hasn't logged yet, or the journal isn't kept on disk split per user, so the service's own user can't read its lines: run scripts/setup/journal.sh on the machine (as an admin).`; @@ -936,7 +943,7 @@ export function readServiceLog(src: LogSource, lines = 200, read: JournalReader // private key, even one that starts above the kept lines) is found whole (#889); each line is redacted again as it // is clipped. const tail = text.split('\n').filter((l) => l.trim()).slice(-(lines + KEY_MARGIN)); - const kept = redactRecords(tail).filter((l) => l.trim()).slice(-lines); + const kept = redactRecords(tail.map(readPart)).filter((l) => l.trim()).slice(-lines); const entries = kept.map((message) => ({ at: null, priority: null, message: clip(message) })); return { source: 'file', unit: src.unit, entries, problem: entries.length ? null : problem }; } diff --git a/src/decision-detail.ts b/src/decision-detail.ts index f040581fa..e2e009a3d 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -15,7 +15,7 @@ import { DEFAULT_REVIEW } from './coordinator/shared.js'; import { isOwner, isWriter, mayAnswerKind, type Role } from './dashboard-auth.js'; import type { DecisionContext } from './decision-context.js'; import { happened, isInvestigation, optionEffects, type DetailKind } from './decision-effects.js'; -import { redactString } from './events/redact.js'; +import { redactCut } from './events/redact.js'; import type { EventPayload, StoredEvent } from './events/types.js'; import type { PrView } from './projection.js'; import { computeLevel, loadMoneyPaths, selfModificationPart } from './review.js'; @@ -143,8 +143,9 @@ export function prSections(body: string | null | undefined): { failWithout: stri const h = heads[i]!; const end = heads.slice(i + 1).find((n) => n.level <= h.level)?.at ?? lines.length; const s = lines.slice(h.at + 1, end).join('\n').trim(); - // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. - return s ? redactString(s).slice(0, 20_000) : null; + // Redacted before it is cut, over the kept part and a margin only (#889): a cut first leaves a token or a key's + // body unfound, and the whole text could be any size. + return s ? redactCut(s, 20_000) : null; }; return { failWithout: section(/fail[- ]?without/i), never: section(/\bnever\b/i) }; } @@ -353,8 +354,9 @@ export async function decisionDiff(root: string, base: string, head: string): Pr execFile('git', ['diff', '--no-color', '--no-ext-diff', `${base}..${head}`], { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { if (err) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); const truncated = stdout.length > MAX; - // Redacted before it is cut (#889): a cut first leaves a token or a key's body unfound. - done({ diff: redactString(stdout).slice(0, MAX), truncated }); + // Redacted before it is cut, over the kept part and a margin only (#889): up to 8 MB of diff, and the redaction is + // synchronous on the dashboard. + done({ diff: redactCut(stdout, MAX), truncated }); }); }); } diff --git a/src/events/redact.ts b/src/events/redact.ts index 3634b4753..08c55c46a 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -66,26 +66,33 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // A dashboard key, wherever its prefix is (after a `-` or a letter too: `key-wld_...`): a match runs to the end of its // run of key characters and the next search starts after it, so linear. [new RegExp(`${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], + // A shape with a distinctive prefix is found wherever the prefix is (#889 round 4): no left boundary, so whatever is + // glued before it (a name, an escape sequence in any encoding, a digit) can't hide it. Each such match runs to the end + // of its run and the next search starts after it: linear. Only shapes with no distinctive prefix keep a boundary. // The token's end: no letter or digit after it, so `ghp_..._backup` is found too (a `\b` there needs a non-word). - [/\b(gh[pousr]_[A-Za-z0-9]{30,})(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], - [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]', 'GitHub token'], + [/gh[pousr]_[A-Za-z0-9]{30,}(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], + [/github_pat_[A-Za-z0-9_]{40,}/g, 'github_pat_[REDACTED]', 'GitHub token'], + [/sk-ant-[A-Za-z0-9_-]{20,}/g, 'sk-[REDACTED]', 'API key (sk-)'], + // Other `sk-` keys: `sk-` alone is no distinctive prefix (`task-`, `disk-`), so after a boundary. [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], // Stripe secret and restricted keys, live and test: random, so with a digit (a long camelCase name after the // prefix, with no digit, is a code identifier). [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v!) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], // Google API keys: AIza and 35 more characters. - [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]', 'Google API key'], + [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], // JSON Web Tokens: a base64url JSON header (so `eyJ`), a payload and a signature (empty when unsigned). // Any further dotted parts go with it (#889 round 3): a JWE has five, and a token may be followed by more. [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], - [/\b(AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], + // And wherever `eyJ` is, with its header bounded (512 characters), so each start reads a bounded stretch first. + [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '[REDACTED JWT]', 'JWT'], + [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], // Slack's token-rotation and refresh tokens (xoxe-..., xoxe.xoxb-...), before the // other Slack tokens (one is inside it); then its app-level token (xapp-, Socket Mode). - [/\bxoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], - [/\bxox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], - [/\bxapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], + [/xoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], + [/xox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], + [/xapp-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xapp-[REDACTED]', 'Slack app-level token'], // Linear personal API keys and OAuth tokens. - [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], + [/lin_(?:api|oauth)_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g, 'lin_[REDACTED]', 'Linear key'], [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], // Tailscale auth keys, API tokens and OAuth client secrets (tskey-auth-..., tskey-api-..., tskey-client-...): the // prefix and the rest of its run, however short (src/tailscale-setup.ts redacted them this way on its own). @@ -96,14 +103,16 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // is read once (its match runs to its END or to the end), and the second rule has one start: linear. [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], - // The scheme is case-insensitive (RFC 9110): `bearer` and `BEARER` too (#889). - [/\b(Bearer|bearer|BEARER)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]', 'bearer token'], + // The scheme is case-insensitive (RFC 9110): any case (#889). `bearer` is a word, so after a boundary. + [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], // HTTP Basic credentials (base64 of user:password; git's extraheader, src/dashboard-auth.ts's client id and secret), // failing closed: any credential-looking value after the scheme. Prose ("Basic Authentication") has no digit, `+`, // `/` or `=`, and isn't a mixed-case run of a base64 length. - [/\b(Basic|basic|BASIC)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/g, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], - // Credentials inside connection strings: keep scheme, user and host. - [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], + [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/gi, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], + // Credentials inside connection strings: keep scheme, user and host. A scheme of up to 32 characters right before + // `://`, from any start (#889 round 4: from `\b` with any length it was quadratic on a long dotted run): each start + // reads at most 33 characters, and each `://` is reached from at most 32 starts, so linear. + [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], // 0x-prefixed 32-byte hex (EVM private keys and the like). [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], // A JSON string value under a credential key (credentialKey: "aws_secret_access_key", "db_password", "x-api-key", @@ -134,13 +143,14 @@ const PATTERNS: [RegExp, Replacement, string][] = [ * after a hyphen. Each starts only where a run starts, or reads a bounded stretch from each start: linear. */ const BROAD: [RegExp, (mask: string) => string, string][] = [ - [/(^|[^A-Za-z0-9_])(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => `$1${k}`, 'GitHub token, cut short'], + // Wherever the prefix is, as in PATTERNS (#889 round 4). + [/(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{8,}/g, (k) => k, 'GitHub token, cut short'], [/sk-ant-[A-Za-z0-9_-]+/g, (k) => k, 'Anthropic key, cut short'], // The root updater's own token rule on main, as it was: where it ends (a GitHub token's run of letters and digits, a // Slack token's before a `_`) is where its next rule saw a boundary (maskUnion reads again there). [/\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-ant-[A-Za-z0-9_-]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g, (k) => k, "the updater's token rule"], - [/(^|[^A-Za-z0-9_])xox[abprs]-[A-Za-z0-9_-]+/g, (k) => `$1${k}`, 'Slack token, cut short'], - [/(^|[^A-Za-z0-9_])eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => `$1${k}`, 'JWT after a hyphen'], + [/xox[abprs]-[A-Za-z0-9_-]+/g, (k) => k, 'Slack token, cut short'], + [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => k, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], // A scheme of up to 32 characters right before `://`, from any start (after `-`, `.`, `+` or a letter too: the // updater's own rule started at any `\b`, a scheme of any length): each start reads at most 33 characters, and each @@ -241,11 +251,24 @@ export interface RedactTextOptions { */ const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', DASHBOARD_KEY_PREFIX]; /** - * What ends in a letter and may sit right before a token (#889): an ANSI escape sequence (a coloured log line, - * `ESC[1m`) and a backslash escape in raw JSON or C-style text (`\n`, `\t`, `\"`, `\u001b`). A cut point goes after - * each. Each ESC or backslash starts one bounded match: linear. + * What ends in a letter or digit and may sit right before a token (#889): an escape sequence (a coloured log line, + * `ESC[1m`, in any of the encodings below) and a backslash escape in raw JSON or C-style text (`\n`, `\t`, `\"`). A + * cut point goes after each. Only the shapes with no distinctive prefix need it (Discord, Bearer, Basic, Stripe, `0x`, + * a plain `sk-`, a JSON value): the others are found wherever their prefix is. Each start reads a bounded stretch, + * but a CSI's parameter run: one scan each, linear. */ -const GLUE = /\u001b(?:\[[0-?]*[ -/]*[@-~]|[@-Z\\-_])|\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])/g; +const GLUE = new RegExp( + [ + // ESC as the byte, as a JSON or C escape (`\u001b`, `\x1b`, `\033`, `\e`) or in caret notation (`^[`), then a whole + // sequence: a CSI with its parameters (`[1m`), an nF one (`(B`), or one character (`7`, `8`, `c`). + String.raw`(?:\u001b|\\u001[bB]|\\x1[bB]|\\033|\\e|\^\[)(?:\[[0-?]*[ -/]*[@-~]|[ -/]+[0-~]|[0-~])`, + // The 8-bit CSI. + String.raw`\u009b[0-?]*[ -/]*[@-~]`, + // A backslash escape in raw JSON or C-style text. + String.raw`\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])`, + ].join('|'), + 'g', +); /** Where a token may start glued to what is before it: after each GLUE match, and after `_` before a known prefix. */ function gluePoints(s: string): number[] { @@ -412,29 +435,29 @@ function maskOnce(s: string, mask: string, keepLines: boolean): string { // What is already a mask stays as it is: a rule that takes in a mask word (`token=[REDACTED]`) adds nothing. const masks: [number, number][] = []; if (mask) for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) masks.push([at, at + mask.length]); - const kept = masks.length ? spans.flatMap(([a, b]) => outside([a, b], masks)) : spans; + const kept = outside(merged(spans), masks); if (!kept.length) return s; - kept.sort((x, y) => x[0] - y[0]); let out = ''; let at = 0; - let open: [number, number] | null = null; - const close = () => { - if (!open) return; - out += s.slice(at, open[0]) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(open[0], open[1]))) : ''); - at = open[1]; - }; for (const [a, b] of kept) { - if (b <= a) continue; - if (open && a <= open[1]) open[1] = Math.max(open[1], b); - else { - close(); - open = [a, b]; - } + out += s.slice(at, a) + mask + (keepLines ? '\n'.repeat(breaks(s.slice(a, b))) : ''); + at = b; } - close(); return out + s.slice(at); } +/** `spans` sorted and merged: no two overlap or touch. */ +function merged(spans: [number, number][]): [number, number][] { + const sorted = spans.filter(([a, b]) => b > a).sort((x, y) => x[0] - y[0]); + const out: [number, number][] = []; + for (const [a, b] of sorted) { + const last = out.at(-1); + if (last && a <= last[1]) last[1] = Math.max(last[1], b); + else out.push([a, b]); + } + return out; +} + /** Every rule on `s` with a boundary marked in at each of `cuts` (sorted); the spans found, mapped back, go in `spans`. */ function readAt(s: string, cuts: number[], spans: [number, number][]) { if (!cuts.length) return; @@ -472,16 +495,23 @@ function readAt(s: string, cuts: number[], spans: [number, number][]) { for (const [a, b] of more) spans.push([back(a), back(b)]); } -/** The parts of [a, b) outside every one of `regions` (sorted, not overlapping). */ -function outside([a, b]: [number, number], regions: [number, number][]): [number, number][] { +/** + * The parts of `spans` outside every one of `regions`, both sorted and not overlapping: one walk over each, with the + * regions' pointer only moving forward (#889 round 4: one loop over every region per span was quadratic). + */ +function outside(spans: [number, number][], regions: [number, number][]): [number, number][] { const out: [number, number][] = []; - let from = a; - for (const [x, y] of regions) { - if (y <= from || x >= b) continue; - if (x > from) out.push([from, x]); - from = Math.max(from, y); + let r = 0; + for (const [a, b] of spans) { + while (r < regions.length && regions[r]![1] <= a) r++; + let from = a; + for (let k = r; k < regions.length && regions[k]![0] < b; k++) { + const [x, y] = regions[k]!; + if (x > from) out.push([from, x]); + from = Math.max(from, y); + } + if (from < b) out.push([from, b]); } - if (from < b) out.push([from, b]); return out; } @@ -509,6 +539,20 @@ export function redactString(s: string, o: RedactTextOptions = {}): string { return out; } +/** What a cut site reads past its cut before redacting (#889 round 4): enough for a token, or a key's END line. */ +export const CUT_MARGIN = 16 * 1024; + +/** + * `s` redacted and then cut to `max` characters, reading only the first `max` + CUT_MARGIN of it (#889 round 4): a + * token across the cut is redacted whole, and a private key across the window's end still fails closed (its BEGIN + * with no END redacts to the end), while a huge input costs no more than the window. Synchronous callers (the + * dashboard) stay responsive. + */ +export function redactCut(s: string, max: number, o: RedactTextOptions = {}): string { + const r = redactString(s.length > max + CUT_MARGIN ? s.slice(0, max + CUT_MARGIN) : s, o); + return r.length > max ? r.slice(0, max) : r; +} + /** * Records shown one after another (a service's journal entries, a log file's lines), redacted as one text so a secret * over several of them (a private key logged a line at a time) is found whole; the same number of records comes back, diff --git a/test/redact-escapes-windows.test.ts b/test/redact-escapes-windows.test.ts new file mode 100644 index 000000000..e7470657d --- /dev/null +++ b/test/redact-escapes-windows.test.ts @@ -0,0 +1,154 @@ +// #889 round 4 (the third adversarial review): +// - A shape with a distinctive prefix is found wherever its prefix is; the rest are found after an escape sequence in +// any encoding (the JSON escape of ESC, ESC 7/8, ESC ( B, ESC c, the 8-bit CSI, `\x1b`, `\033`, `\e`, `^[`). +// - No redaction site costs more than its window: the default URL-password rule is linear on long dotted runs, and the +// diff view, a chat attachment, a PR section and the log views redact only what they keep plus a margin. +// - Broad mode's mask subtraction is linear; Bearer and Basic match in any case. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { redactString } from '../src/events/redact.js'; +import { readServiceLog } from '../src/dashboard.js'; +import { decisionDiff, prSections } from '../src/decision-detail.js'; +import { attachmentPrompt, ATTACHMENT_DEFAULT_CAPS, storeUpload } from '../src/chat-attachments.js'; +import { safeLine } from '../src/gh-login.js'; +import { scrub as benchScrub } from '../scripts/bench/cli.js'; +import { childEnv } from '../src/os/index.js'; +import { BRAND } from '../src/brand.js'; +import { repoRoot } from './helpers.js'; +import { leaks, run, SHAPES } from './token-standins.js'; + +const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); +const updater = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`)) as { redact(text: string): string }; +const ESC = '\u001b'; +const BS = '\\'; +const broad = { broad: true } as Parameters[1]; + +/** The five paths the review drove: default, broad, the root updater, gh-login and the bench. */ +const PATHS: [string, (t: string) => string][] = [ + ['default', (t) => redactString(t)], + ['broad', (t) => redactString(t, broad)], + ['updater', (t) => updater.redact(t)], + ['gh-login', (t) => safeLine(t)], + ['bench', (t) => benchScrub(t, {}, 100_000)], +]; + +test('a token right after an escape sequence in any encoding is found, every token shape on every path (R1-R4)', () => { + const escapes: [string, string][] = [ + ['JSON-escaped ESC, CSI', `${BS}u001b[1m`], + ['JSON-escaped ESC, CSI with parameters', `${BS}u001B[32;1m`], + ['ESC 7 (save cursor)', `${ESC}7`], + ['ESC 8 (restore cursor)', `${ESC}8`], + ['ESC ( B (charset)', `${ESC}(B`], + ['ESC c (reset)', `${ESC}c`], + ['8-bit CSI', '\u009b1m'], + ['8-bit CSI with parameters', '\u009b32;1m'], + ['C hex escape, CSI', `${BS}x1b[1m`], + ['C octal escape, CSI', `${BS}033[1m`], + ['\\e, CSI', `${BS}e[1m`], + ['caret notation, CSI', '^[[1m'], + ]; + const missed: string[] = []; + for (const s of SHAPES.filter((x) => x.token && !x.value.includes('\n'))) { + for (const [how, esc] of escapes) { + for (const [path, out] of PATHS) { + if (path === 'gh-login' && !s.kind.startsWith('GitHub')) continue; + if (leaks(out(`log: ${esc}${s.value} done`), s.secret)) missed.push(`${s.what}, ${how}, via ${path}`); + } + } + } + assert.deepEqual(missed, []); +}); + +test('a shape with a distinctive prefix is found after any character, a digit or a letter included', () => { + const missed: string[] = []; + const distinctive = SHAPES.filter((x) => x.token && /^(?:gh[pousr]_|github_pat_|sk-ant-|xox|xapp-|lin_|tskey-|wld_|AKIA|ASIA|AIza|eyJ)/.test(x.value)); + assert.ok(distinctive.length >= 20, `${distinctive.length} stand-ins with a distinctive prefix`); + for (const s of distinctive) for (const before of ['7', 'x', 'Z', '_', '%5F', '_', 'm']) if (leaks(redactString(`${before}${s.value} end`), s.secret)) missed.push(`${s.what} after ${before}`); + assert.deepEqual(missed, []); +}); + +test('Bearer and Basic match in any case, plain and in a JSON header, in default mode (R9)', () => { + const tok = run(40, 'bR5x'); + const cred = Buffer.from(`x-access-token:${run(30)}`).toString('base64'); + for (const s of [`authorization: bEaReR ${tok}`, `{"Authorization":"bEarer ${tok}"}`]) assert.ok(!leaks(redactString(s), tok), s); + for (const s of [`Authorization: bAsIc ${cred}`, `{"Authorization":"basiC ${cred}"}`]) assert.ok(!leaks(redactString(s), cred), s); +}); + +/** Long dotted runs (R6, R8): a URL scheme's characters, a dotted eyJ chain, and secret names among mask words (R7). */ +const HOSTILE = (n: number) => ['ab.'.repeat(n / 3), 'ab-c+d.'.repeat(n / 7), `eyJ${'abcdefghij.'.repeat(n / 11)}`, '[hidden] token=x '.repeat(n / 17), `${'a.'.repeat(n / 4)}://x:y@h`]; +const timed = (f: () => unknown) => { + const t0 = performance.now(); + f(); + return performance.now() - t0; +}; + +test('redactString, default and broad, stays linear on long dotted runs and many mask words (R6, R7, R8)', () => { + const slow: string[] = []; + for (const s of HOSTILE(400_000)) { + for (const [what, f] of [['default', () => redactString(s)], ['broad', () => redactString(s, broad)], ['broad [hidden]', () => redactString(s, { broad: true, mask: '[hidden]' } as Parameters[1])]] as const) { + const ms = timed(f); + if (ms > 3000) slow.push(`${what} on ${JSON.stringify(s.slice(0, 12))}…: ${Math.round(ms)} ms`); + } + } + assert.deepEqual(slow, []); +}); + +test('each cut site redacts only its window: a PR section, a chat attachment, the log views (journal and file), the diff view', async () => { + const big = HOSTILE(1_000_000).join('\n'); + const slow: string[] = []; + const check = async (what: string, f: () => unknown) => { + const t0 = performance.now(); + await f(); + const ms = performance.now() - t0; + if (ms > 3000) slow.push(`${what}: ${Math.round(ms)} ms`); + }; + await check('PR section', () => prSections(`## Fail-without\n${big}\n\n## Next\n`)); + const s = scratch('attach-'); + async function* once(b: string) { + yield Buffer.from(b); + } + const m = await storeUpload({ stateDir: s, name: 'log.txt', body: once(big), caps: { ...ATTACHMENT_DEFAULT_CAPS }, by: 'example-collaborator', gitleaks: null, anyway: true }); + await check('chat attachment', () => attachmentPrompt(s, [m.id], scratch('bundle-'))); + // 300 journal entries of 48 KB (journald's LineMax): what the view reads at most. + const entry = 'ab.'.repeat(16_000); + const journal = () => ({ status: 0, stderr: '', stdout: Array.from({ length: 300 }, (_, i) => JSON.stringify({ MESSAGE: entry, PRIORITY: '6', __REALTIME_TIMESTAMP: String(1_767_225_600_000_000 + i) })).join('\n') }); + await check('journal view', () => readServiceLog({ unit: 'coordinator.service', file: null }, 200, journal)); + const file = join(scratch('svc-log-'), 'c.log'); + writeFileSync(file, `${Array.from({ length: 300 }, () => entry).join('\n')}\n`); + await check('file view', () => readServiceLog({ unit: null, file }, 200)); + // The diff view: a commit adding 1 MB of dotted runs. + const repo = scratch('diff-repo-'); + const env = { ...childEnv(), GIT_CONFIG_GLOBAL: join(repo, '.no-global'), GIT_CONFIG_NOSYSTEM: '1' }; + const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, encoding: 'utf8', env }).trim(); + git('init', '-q', '-b', 'main'); + for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); + writeFileSync(join(repo, 'a.txt'), 'a\n'); + git('add', '-A'); + git('commit', '-q', '-m', 'base'); + const base = git('rev-parse', 'HEAD'); + writeFileSync(join(repo, 'a.txt'), `${big}\n`); + git('add', '-A'); + git('commit', '-q', '-m', 'big'); + const head = git('rev-parse', 'HEAD'); + let diff: Awaited> | null = null; + await check('diff view', async () => { + diff = await decisionDiff(repo, base, head); + }); + assert.ok(diff && 'truncated' in diff! && (diff as { truncated: boolean }).truncated, 'the diff was cut'); + assert.deepEqual(slow, []); +}); + +test('a key cut at a window edge still fails closed: its body is not shown', () => { + // 400 body lines (26 KB), so its END is past the cut and the margin. + const body = Array.from({ length: 400 }, (_, i) => run(64, `W${i}nd0wKey${i}`)); + const key = [['-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'].join(' '), ...body, ['-----END', 'RSA', 'PRIVATE', 'KEY-----'].join(' ')].join('\n'); + // The key starts just before the PR section's 20,000-character cut, its END far past the window. + const section = prSections(`## Fail-without\n${'x '.repeat(9_990)}\n${key}\n${'y '.repeat(20_000)}\n\n## Next\n`).failWithout!; + assert.equal(body.filter((b) => leaks(section, b)).length, 0); +}); diff --git a/test/redact.test.ts b/test/redact.test.ts index de67b1072..59b3056fc 100644 --- a/test/redact.test.ts +++ b/test/redact.test.ts @@ -277,9 +277,11 @@ test('every pattern runs in linear time on 100 KB of hostile input', () => { ['" + a', run('a', '"')], ['" + a_', run('a_', '"', '":"')], ['"a', run('"a')], ['"x-api-key":"', run('"x-api-key":"')], ['"a_b_key": + value', run('a_b_', '"', `key":"${'Aa1'.repeat(6)}"`)], ['//a.lan', run('//a.lan')], ['/a.lan', run('/a.lan')], ['a.Internal ', run('a.Internal ')], ['aB.local ', run('aB.local ')], ['this.a.internal', run('a.', 'this.', 'internal')], + // Long runs of letters with `.`, `-` or `+` (a URL scheme's characters), and with `://` at the end (#889 round 4). + ['ab.', run('ab.')], ['ab-', run('ab-')], ['ab+', run('ab+')], ['ab. + ://', run('ab.', '', '://x:y@')], ['a.b-c+', run('a.b-c+')], ]; - // The URL-password pattern predates #644 and is quadratic on these shapes too: a follow-up, not this change. - const KNOWN_SLOW = new Set(['password in a URL']); + // None: the URL-password pattern, quadratic on these shapes before, takes a bounded scheme since #889 round 4. + const KNOWN_SLOW = new Set([]); const slow: string[] = []; for (const { kind, re, rep } of REDACTION_PATTERNS) { if (KNOWN_SLOW.has(kind)) continue; From a5c86ec2b4853997ede3a683d8f5785b3b2cc208 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 01:27:33 -0500 Subject: [PATCH 6/9] redaction: a read window never shows its raw end, JWT header unbounded, unscannable text fails closed (#889) Round 5, on the fourth adversarial review (#921): - readWindow: a cut site's read window (redactCut's max + 16 KB, a log record's 6,096) ends at its last whitespace, or is dropped when it has none. Redaction earlier in the window (an END line with no BEGIN, a closed key over 16 KB) could shrink the text until the window's raw end, a token cut too short for its rule, landed inside what is shown. - A JWT is read from the start of its header's run with no length bound, and a function finds `eyJ` in the run, so a header over 512 characters glued after a letter, digit, `-` or `%3D` is found. Each start is a run start: linear. - redactString catches V8's RangeError on a run of millions of characters and returns a fixed placeholder (the line count kept for redactRecords) instead of throwing. Test (fails without): test/redact-window-edge.test.ts, 5 of 5 failing at 88ec30b7. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 2 +- scripts/machine/worklane-update.cjs | 34 ++++++- src/dashboard.ts | 5 +- src/events/redact.ts | 43 ++++++++- test/redact-window-edge.test.ts | 123 +++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 test/redact-window-edge.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index 31464517d..6e2055c20 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -7,5 +7,5 @@ - **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. A shape with a distinctive prefix (`ghp_`, `github_pat_`, `sk-ant-`, `xoxb-`, `lin_api_`, `AKIA`, `AIza`, `eyJ`, `wld_`, ...) is found wherever its prefix is, whatever is glued before it. The other shapes (a Discord bot token, a Bearer or Basic credential in any case, a Stripe key, a plain `sk-` key) are found after a name glued with `_`, an escape sequence (ESC as a byte, as `\u001b`, `\x1b`, `\033`, `\e` or `^[`, and the 8-bit CSI) or a JSON escape (`\n`). Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. - **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. This covers PEM keys (upper-case BEGIN and END lines with five dashes) and PGP's private key block. Not covered: a key whose BEGIN and END are both outside a window (only its middle lines in view), and other formats (an ssh.com SSH2 key, a lower-case PEM line, a PuTTY `.ppk` file). - **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. -- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. The URL-password rule no longer slows down on a long dotted line. +- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. Each window ends at its last whitespace, so a token cut by the window's end is never shown. A text the redactor can't scan (one run of millions of characters) becomes `[REDACTED: too large to scan]`. The URL-password rule no longer slows down on a long dotted line. - A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 41193ade0..08abdee40 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -115,10 +115,11 @@ const sharedRedact = (() => { const exports = {}; "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.CUT_MARGIN = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; +exports.CUT_MARGIN = exports.UNSCANNABLE = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; exports.secretKinds = secretKinds; exports.redactString = redactString; exports.redactCut = redactCut; +exports.readWindow = readWindow; exports.redactRecords = redactRecords; exports.redact = redact; const credentialShaped = (v) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); @@ -153,8 +154,14 @@ const PATTERNS = [ [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], - [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], - [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '[REDACTED JWT]', 'JWT'], + [ + /(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{11,})\.([A-Za-z0-9_-]{8,})\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, + (m, pre, head) => { + const at = head.indexOf('eyJ'); + return at !== -1 && head.length - at >= 11 ? `${pre}${head.slice(0, at)}[REDACTED JWT]` : m; + }, + 'JWT', + ], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], [/xoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], [/xox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], @@ -446,7 +453,18 @@ function maskUnion(s, mask, keepLines) { } return out; } +exports.UNSCANNABLE = '[REDACTED: too large to scan]'; function redactString(s, o = {}) { + try { + return redactOrThrow(s, o); + } + catch (e) { + if (e instanceof RangeError) + return exports.UNSCANNABLE + (o.keepLines ? '\n'.repeat(breaks(s)) : ''); + throw e; + } +} +function redactOrThrow(s, o) { const put = o.keepLines ? applyKeepingLines : apply; const secrets = (t) => PATTERNS.reduce((y, [re, rep]) => put(y, re, rep), t); let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : glued(secrets(s), secrets); @@ -456,9 +474,17 @@ function redactString(s, o = {}) { } exports.CUT_MARGIN = 16 * 1024; function redactCut(s, max, o = {}) { - const r = redactString(s.length > max + exports.CUT_MARGIN ? s.slice(0, max + exports.CUT_MARGIN) : s, o); + const r = redactString(s.length > max + exports.CUT_MARGIN ? readWindow(s, max + exports.CUT_MARGIN) : s, o); return r.length > max ? r.slice(0, max) : r; } +function readWindow(s, end) { + if (s.length <= end) + return s; + for (let i = end - 1; i >= 0; i--) + if (/\s/.test(s[i])) + return s.slice(0, i + 1); + return ''; +} function redactRecords(records, o = {}) { const lines = redactString(records.join('\n'), { ...o, keepLines: true }).split('\n'); const out = []; diff --git a/src/dashboard.ts b/src/dashboard.ts index 0e00cdecc..e09cbf481 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -36,7 +36,7 @@ import { EventCache, type OpenRead } from './events/cache.js'; import { compress, compressibleType, MIN_COMPRESS_BYTES, mayCompressApi, requestEncoding, StaticCompressed, type Encoding } from './dashboard-compress.js'; import { dashboardProof, NONCE, PROOF_PATH } from './dashboard-proof.js'; import { localDay } from './local-day.js'; -import { DASHBOARD_KEY_PREFIX, redactCut, redactRecords, redactString } from './events/redact.js'; +import { DASHBOARD_KEY_PREFIX, readWindow, redactCut, redactRecords, redactString } from './events/redact.js'; import type { EventPayload, StoredEvent } from './events/types.js'; import { acceptGit, acceptMerge, protectionView, type AcceptGit, type AcceptTarget } from './main-watch.js'; import { deployGateView } from './deploy-gate/gate.js'; @@ -889,7 +889,8 @@ const journalctl: JournalReader = (unit, lines) => { const RECORD_SHOWN = 2000; const RECORD_READ = RECORD_SHOWN + 4096; /** A record cut to what is read of it: a huge journal line costs no more than that (the redaction is synchronous). */ -const readPart = (s: string) => (s.length > RECORD_READ ? s.slice(0, RECORD_READ) : s); +// It ends at the record's last whitespace in that part (readWindow, #889 round 5): its raw end is never shown. +const readPart = (s: string) => readWindow(s, RECORD_READ); /** A message cut to 2,000 characters, redacted first: a cut made before would leave a long secret's body unfound (#889). */ const clip = (s: string) => { const r = redactCut(s, RECORD_SHOWN); diff --git a/src/events/redact.ts b/src/events/redact.ts index 08c55c46a..e2e780bcb 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -82,9 +82,17 @@ const PATTERNS: [RegExp, Replacement, string][] = [ [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], // JSON Web Tokens: a base64url JSON header (so `eyJ`), a payload and a signature (empty when unsigned). // Any further dotted parts go with it (#889 round 3): a JWE has five, and a token may be followed by more. - [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '$1[REDACTED JWT]', 'JWT'], - // And wherever `eyJ` is, with its header bounded (512 characters), so each start reads a bounded stretch first. - [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, '[REDACTED JWT]', 'JWT'], + // Read from the start of the header's run of base64url characters, with no bound on its length (an `x5c` chain makes + // a header several KB, #889 round 5); a function finds `eyJ` in that run, so a token glued after a letter, a digit or + // `-` is found from its `eyJ` on. Each start is a run's start: linear. + [ + /(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{11,})\.([A-Za-z0-9_-]{8,})\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, + (m, pre, head) => { + const at = head!.indexOf('eyJ'); + return at !== -1 && head!.length - at >= 11 ? `${pre}${head!.slice(0, at)}[REDACTED JWT]` : m; + }, + 'JWT', + ], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], // Slack's token-rotation and refresh tokens (xoxe-..., xoxe.xoxb-...), before the // other Slack tokens (one is inside it); then its app-level token (xapp-, Socket Mode). @@ -529,7 +537,21 @@ function maskUnion(s: string, mask: string, keepLines: boolean): string { return out; } +/** What a text becomes when it can't be scanned (#889 round 5): shown as nothing of it. */ +export const UNSCANNABLE = '[REDACTED: too large to scan]'; + export function redactString(s: string, o: RedactTextOptions = {}): string { + try { + return redactOrThrow(s, o); + } catch (e) { + // V8's backtracking stack overflows on one run of millions of characters after a prefix (`RangeError: Maximum + // call stack size exceeded`): fail closed, with the text's line count kept for redactRecords. + if (e instanceof RangeError) return UNSCANNABLE + (o.keepLines ? '\n'.repeat(breaks(s)) : ''); + throw e; + } +} + +function redactOrThrow(s: string, o: RedactTextOptions): string { const put = o.keepLines ? applyKeepingLines : apply; // The broad mode masks the union of every rule's spans (maskUnion); the default mode applies the shapes in turn, // each with its own marker, then again from each glue point (glued). @@ -549,10 +571,23 @@ export const CUT_MARGIN = 16 * 1024; * dashboard) stay responsive. */ export function redactCut(s: string, max: number, o: RedactTextOptions = {}): string { - const r = redactString(s.length > max + CUT_MARGIN ? s.slice(0, max + CUT_MARGIN) : s, o); + const r = redactString(s.length > max + CUT_MARGIN ? readWindow(s, max + CUT_MARGIN) : s, o); return r.length > max ? r.slice(0, max) : r; } +/** + * The first `end` characters of `s` as a window to redact, ending at its last whitespace (#889 round 5): the raw end of + * a window is never shown. A token cut by the window's end is too short for its rule, and redaction earlier in the + * window (a key, an END line with no BEGIN) can shrink the text so far that the cut lands inside what is shown. No + * token, URL user info or credential value holds whitespace, so ending at one drops the cut run whole; a window with no + * whitespace at all is dropped (empty). A private key cut this way still fails closed by its BEGIN. + */ +export function readWindow(s: string, end: number): string { + if (s.length <= end) return s; + for (let i = end - 1; i >= 0; i--) if (/\s/.test(s[i]!)) return s.slice(0, i + 1); + return ''; +} + /** * Records shown one after another (a service's journal entries, a log file's lines), redacted as one text so a secret * over several of them (a private key logged a line at a time) is found whole; the same number of records comes back, diff --git a/test/redact-window-edge.test.ts b/test/redact-window-edge.test.ts new file mode 100644 index 000000000..f410144b6 --- /dev/null +++ b/test/redact-window-edge.test.ts @@ -0,0 +1,123 @@ +// #889 round 5 (the fourth adversarial review): +// - A cut site never shows the raw end of its read window. Redaction earlier in the window (an END line with no BEGIN, +// or a closed key over 16 KB) can shrink the text so far that the window's cut lands inside what is shown; a token +// cut there is too short for its rule. Each window now ends at its last whitespace (readWindow). +// - A JWT whose header is longer than 512 characters (an x5c chain), glued after a letter, a digit, `-` or `%3D`. +// - A run of millions of characters after a prefix overflows V8's backtracking stack: redactString fails closed. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as redactModule from '../src/events/redact.js'; +import { redactString } from '../src/events/redact.js'; +import { readServiceLog } from '../src/dashboard.js'; +import { BRAND } from '../src/brand.js'; +import { repoRoot } from './helpers.js'; +import { leaks, run } from './token-standins.js'; + +const scratch = (p: string) => mkdtempSync(join(tmpdir(), p)); +const updater = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`)) as { redact(text: string): string }; +const broad = { broad: true } as Parameters[1]; +const MARGIN = 16 * 1024; +const BEGIN = ['-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const END = ['-----END', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); + +/** + * Tokens to put at a window's edge, each just long enough for its rule, so that cut by one character it is too short + * (or, for the URL password, loses its `@`); each with the part that must not be shown. + */ +const TOKENS: [string, string, string][] = [ + ['a dashboard key', `${'wld'}_${run(32, 'Wd9kQ2-x_')}`, run(32, 'Wd9kQ2-x_').slice(0, 30)], + ['a GitHub token', `${'ghp'}_${run(30)}`, run(30).slice(0, 29)], + ['a URL password', `https://app:${run(24, 'Pw7xQ2zL')}@`, run(24, 'Pw7xQ2zL')], +]; + +/** Text of `n` characters that shrinks when redacted: everything up to an END line with no BEGIN, or a closed key. */ +const SHRINKERS: [string, (n: number, newline: string) => string][] = [ + ['an END line with no BEGIN', (n) => `${'x '.repeat(Math.ceil((n - END.length - 1) / 2)).slice(0, n - END.length - 1)}${END} `], + [ + 'a closed key', + (n, nl) => { + const lines = Math.floor((n - BEGIN.length - END.length - 4) / (64 + nl.length)); + const key = [BEGIN, ...Array.from({ length: lines }, (_, i) => run(64, `K${i}b0dy${i}`)), END].join(nl); + return `${key}${' '.repeat(n - key.length)}`; + }, + ], +]; + +/** A text whose read window (its first `end` characters) cuts `token` before its last character, after `shrinker`. */ +function atEdge(end: number, shrink: string, token: string): string { + const start = end - (token.length - 1); + const filler = 'w '.repeat(Math.ceil((start - shrink.length) / 2)).slice(0, start - shrink.length - 1); + return `${shrink}${filler} ${token} and the rest of the text`; +} + +test('redactCut never shows a token cut by its window, after an END with no BEGIN or a closed key over 16 KB (2,000, 20,000, 512 KB)', () => { + const missed: string[] = []; + for (const max of [2_000, 20_000, 512 * 1024]) { + for (const [how, shrinker] of SHRINKERS) { + for (const [what, token, secret] of TOKENS) { + const s = atEdge(max + MARGIN, shrinker(MARGIN + 600, '\n'), token); + assert.ok(s.length > max + MARGIN, 'the text is longer than the window'); + const out = redactModule.redactCut(s, max); + if (leaks(out, secret)) missed.push(`${what} after ${how}, max ${max}`); + } + } + } + assert.deepEqual(missed, []); +}); + +test('the log views never show a token cut by a record\'s read part, after an END with no BEGIN or a closed key (journal and file)', () => { + const READ = 2000 + 4096; + const missed: string[] = []; + for (const [how, shrinker] of SHRINKERS) { + for (const [what, token, secret] of TOKENS) { + // The journal: one entry, a closed key with real line breaks in it. + const entry = atEdge(READ, shrinker(4300, '\n'), token); + const journal = () => ({ status: 0, stderr: '', stdout: JSON.stringify({ MESSAGE: entry, PRIORITY: '6', __REALTIME_TIMESTAMP: '1767225600000000' }) }); + if (leaks(JSON.stringify(readServiceLog({ unit: 'coordinator.service', file: null }, 10, journal).entries), secret)) missed.push(`${what} after ${how}, journal`); + // The file: one line, a closed key with its line breaks written as `\n` (a JSON log line). + const file = join(scratch('svc-log-'), 'c.log'); + writeFileSync(file, `${atEdge(READ, shrinker(4300, '\\n'), token)}\n`); + if (leaks(JSON.stringify(readServiceLog({ unit: null, file }, 10).entries), secret)) missed.push(`${what} after ${how}, file`); + } + } + assert.deepEqual(missed, []); +}); + +test('readWindow ends a window at its last whitespace, and drops a window with none', () => { + assert.equal(typeof redactModule.readWindow, 'function'); + assert.equal(redactModule.readWindow('abc def ghi', 9), 'abc def '); + assert.equal(redactModule.readWindow('abcdefghij', 5), ''); + assert.equal(redactModule.readWindow('short', 10), 'short'); +}); + +test('a JWT with a long header (an x5c chain) is found glued after a letter, a digit, `-` or `%3D`, default and broad', () => { + const missed: string[] = []; + for (const len of [600, 3000]) { + const sig = run(43, 'S1gn4tUr3'); + const jwt = `${'eyJ'}${run(len, 'hdrX5cChain9')}.${'eyJ'}${run(40, 'pAyL0ad')}.${sig}`; + for (const before of ['x', '7', '-', '%3D', '_', ' ']) { + for (const [mode, out] of [['default', redactString(`token ${before}${jwt} end`)], ['broad', redactString(`token ${before}${jwt} end`, broad)]] as const) { + if (leaks(out, sig)) missed.push(`header ${len} after ${JSON.stringify(before)}, ${mode}`); + } + } + } + assert.deepEqual(missed, []); + // A dotted run with no eyJ is left as it was. + for (const s of ['config.settings.production.example', 'abcdefghijklmnop.qrstuvwxyz.abc']) assert.equal(redactString(s), s); +}); + +test('a run of millions of characters after a prefix fails closed instead of throwing (default, broad, the updater)', () => { + const huge = `${'ghp'}_${'a'.repeat(6 * 1024 * 1024)}`; + for (const [what, f] of [['default', () => redactString(huge)], ['broad', () => redactString(huge, broad)], ['updater', () => updater.redact(huge)]] as const) { + let out = ''; + assert.doesNotThrow(() => { + out = f(); + }, what); + assert.ok(!out.includes('a'.repeat(100)), `${what}: nothing of it is shown`); + } +}); From 1eef36254c0019087d88a4203f4ada50aa9d8a8b Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 02:31:03 -0500 Subject: [PATCH 7/9] redaction: default mode reads every rule on the original text, JWT in any dotted part, no stack overflow (#889) Round 6, on the round-3+4 and round-5 delta checks (#921): - Default mode is order-free: every shape runs on the text as it was (and from each glue point), and each run of overlapping spans is replaced once; a span that covers its run keeps its rule's marker, otherwise `[REDACTED]`. Then again on the result, up to 3 passes, as broad mode does. One rule's match can no longer take in the text another rule starts from (`wld_-redis://u:@`, `.Bearer `, `xoxb-...-lin_api_`). - The orphan-END rule decides in a function: an END line counts only when no BEGIN opened it, since the rules now read the text with each closed key still in it. - The JWT rule reads every dotted part of a run as a possible header (`sess_0123456789.`), and steps over its first two parts with a lookahead and backreference, so a plain run of millions of characters no longer overflows V8's backtracking stack (it is left as it was, as on main). - secretKinds fails closed on a RangeError: it names the text `text too large to scan`, so an attachment is refused (409) until attached anyway. - The dashboard's duplicate doc comment is gone. test/redact-default-union.test.ts's fixtures are registered never-shrink. Tests (fail without): test/redact-default-union.test.ts and test/redact-unscannable.test.ts, 6 of 6 failing at a5c86ec2. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 3 +- scripts/machine/worklane-update.cjs | 149 +++++++++++------- src/dashboard.ts | 1 - src/events/redact.ts | 202 +++++++++++++++++-------- src/merge-policy-weakening.ts | 3 + test/redact-default-union.test.ts | 177 ++++++++++++++++++++++ test/redact-unscannable.test.ts | 45 ++++++ 7 files changed, 463 insertions(+), 117 deletions(-) create mode 100644 test/redact-default-union.test.ts create mode 100644 test/redact-unscannable.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index 6e2055c20..fcbeb5803 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -6,6 +6,7 @@ - **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. A shape with a distinctive prefix (`ghp_`, `github_pat_`, `sk-ant-`, `xoxb-`, `lin_api_`, `AKIA`, `AIza`, `eyJ`, `wld_`, ...) is found wherever its prefix is, whatever is glued before it. The other shapes (a Discord bot token, a Bearer or Basic credential in any case, a Stripe key, a plain `sk-` key) are found after a name glued with `_`, an escape sequence (ESC as a byte, as `\u001b`, `\x1b`, `\033`, `\e` or `^[`, and the 8-bit CSI) or a JSON escape (`\n`). Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. - **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. This covers PEM keys (upper-case BEGIN and END lines with five dashes) and PGP's private key block. Not covered: a key whose BEGIN and END are both outside a window (only its middle lines in view), and other formats (an ssh.com SSH2 key, a lower-case PEM line, a PuTTY `.ppk` file). +- **The rules no longer run one after another.** Every rule reads the text as it was, and each run of what they find is replaced once, so one rule's match can't take in the text another rule starts from. Before, a dashboard key, a JWT, a Slack or Tailscale token or a bearer value written right before a URL with a password, a bearer token or a Linear key could hide it from the next rule (`wld_-redis://app:@…`). A JWT is found in any dotted part of a run (`sess_0123456789.`). Where one rule's find covers the others, it shows that rule's marker (a bearer token that is an `sk-` key now shows as `Bearer [REDACTED]`); where finds only overlap, the run shows `[REDACTED]`. A test holds the default mode to the previous release's rules on composed input. - **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. -- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. Each window ends at its last whitespace, so a token cut by the window's end is never shown. A text the redactor can't scan (one run of millions of characters) becomes `[REDACTED: too large to scan]`. The URL-password rule no longer slows down on a long dotted line. +- **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. Each window ends at its last whitespace, so a token cut by the window's end is never shown. A text the redactor can't scan (a token prefix followed by a run of millions of characters) becomes `[REDACTED: too large to scan]`, and a chat attachment like that is refused as "text too large to scan" until it is attached anyway. The URL-password rule no longer slows down on a long dotted line. - A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 08abdee40..49e701fd6 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -115,7 +115,7 @@ const sharedRedact = (() => { const exports = {}; "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.CUT_MARGIN = exports.UNSCANNABLE = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; +exports.CUT_MARGIN = exports.UNSCANNABLE = exports.UNSCANNABLE_KIND = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; exports.secretKinds = secretKinds; exports.redactString = redactString; exports.redactCut = redactCut; @@ -146,6 +146,32 @@ exports.DASHBOARD_KEY_PREFIX = 'wld_'; const DASHBOARD_KEY_BODY = '[A-Za-z0-9_-]{32,}'; const isDashboardKey = (key) => new RegExp(`^${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}$`).test(key); exports.isDashboardKey = isDashboardKey; +const KEY_LINE = /-----(BEGIN|END) [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/g; +function orphanEnd(m) { + let open = false; + let cut = -1; + KEY_LINE.lastIndex = 0; + for (let k = KEY_LINE.exec(m); k; k = KEY_LINE.exec(m)) { + if (k[1] === 'BEGIN') + open = true; + else if (open) + open = false; + else + cut = k.index + k[0].length; + } + return cut === -1 ? m : `[REDACTED PRIVATE KEY]${m.slice(cut)}`; +} +function jwtIn(m, pre) { + const parts = m.slice(pre.length).split('.'); + let before = pre; + for (let i = 0; i + 2 < parts.length; i++) { + const at = parts[i].indexOf('eyJ'); + if (at !== -1 && parts[i].length - at >= 11 && parts[i + 1].length >= 8) + return `${before}${parts[i].slice(0, at)}[REDACTED JWT]`; + before += `${parts[i]}.`; + } + return m; +} const PATTERNS = [ [new RegExp(`${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], [/gh[pousr]_[A-Za-z0-9]{30,}(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], @@ -154,14 +180,7 @@ const PATTERNS = [ [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], - [ - /(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{11,})\.([A-Za-z0-9_-]{8,})\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, - (m, pre, head) => { - const at = head.indexOf('eyJ'); - return at !== -1 && head.length - at >= 11 ? `${pre}${head.slice(0, at)}[REDACTED JWT]` : m; - }, - 'JWT', - ], + [/(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]+))\2\.(?=([A-Za-z0-9_-]+))\3\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, (m, pre) => jwtIn(m, pre), 'JWT'], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], [/xoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], [/xox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], @@ -170,7 +189,7 @@ const PATTERNS = [ [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]', 'Slack webhook'], [/tskey-[A-Za-z0-9_-]+/g, 'tskey-[REDACTED]', 'Tailscale key'], [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], - [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], + [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, orphanEnd, 'private key'], [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/gi, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], @@ -251,25 +270,6 @@ function freeMark(s) { return String.fromCharCode(c); return null; } -function glued(s, f) { - const cuts = gluePoints(s); - if (!cuts.length) - return s; - const mark = freeMark(s); - if (mark === null) { - let out = s.slice(0, cuts[0]); - for (const [i, c] of cuts.entries()) - out += f(s.slice(c, cuts[i + 1] ?? s.length)); - return out; - } - let marked = ''; - let from = 0; - for (const c of cuts) { - marked += s.slice(from, c) + mark; - from = c; - } - return f(marked + s.slice(from)).split(mark).join(''); -} const expand = (rep, m, groups) => rep.replace(/\$(\d|\$|&)/g, (_, k) => (k === '$' ? '$' : k === '&' ? m : String(groups[Number(k) - 1] ?? ''))); const breaks = (s) => s.split('\n').length - 1; function applyKeepingLines(s, re, rep) { @@ -314,12 +314,20 @@ function outsideMachineBlocks(s, f) { } return out + f(s.slice(from)); } +exports.UNSCANNABLE_KIND = 'text too large to scan'; function secretKinds(s) { const kinds = new Set(); - for (const [re, rep, kind] of PATTERNS) { - re.lastIndex = 0; - if (apply(s, re, rep) !== s) - kinds.add(kind); + try { + for (const [re, rep, kind] of PATTERNS) { + re.lastIndex = 0; + if (apply(s, re, rep) !== s) + kinds.add(kind); + } + } + catch (e) { + if (e instanceof RangeError) + return [exports.UNSCANNABLE_KIND]; + throw e; } return [...kinds].sort(); } @@ -330,7 +338,7 @@ const globalOf = (re) => { globals.set(re, (g = re.flags.includes('g') ? re : new RegExp(re.source, `${re.flags}g`))); return g; }; -function spansOf(t, re, rep, out) { +function spansOf(t, re, rep, rank, out) { const g = globalOf(re); g.lastIndex = 0; for (let m = g.exec(t); m; m = g.exec(t)) { @@ -347,22 +355,60 @@ function spansOf(t, re, rep, out) { while (b < whole.length - a && b < r.length - a && whole[whole.length - 1 - b] === r[r.length - 1 - b]) b++; if (whole.length - b > a) - out.push([m.index + a, m.index + whole.length - b]); + out.push([m.index + a, m.index + whole.length - b, r.slice(a, r.length - b), rank]); } } -const BROAD_RULES = [...PATTERNS.map(([re, rep]) => [re, rep]), ...BROAD.map(([re, rep]) => [re, rep('[REDACTED]')])]; -function maskOnce(s, mask, keepLines) { +const SHAPE_RULES = PATTERNS.map(([re, rep]) => [re, rep]); +const BROAD_RULES = [...SHAPE_RULES, ...BROAD.map(([re, rep]) => [re, rep('[REDACTED]')])]; +function allSpans(s, rules) { const spans = []; - for (const [re, rep] of BROAD_RULES) - spansOf(s, re, rep, spans); + rules.forEach(([re, rep], i) => spansOf(s, re, rep, i, spans)); + readAt(s, gluePoints(s), rules, spans); + return spans; +} +function replaceOnce(s, keepLines) { + const spans = allSpans(s, SHAPE_RULES); + if (!spans.length) + return s; + spans.sort((x, y) => x[0] - y[0] || y[1] - x[1] || x[3] - y[3]); + let out = ''; + let at = 0; + for (let i = 0; i < spans.length;) { + const a = spans[i][0]; + let b = spans[i][1]; + let j = i + 1; + for (; j < spans.length && spans[j][0] < b; j++) + b = Math.max(b, spans[j][1]); + let pick; + for (let k = i; k < j; k++) + if (spans[k][0] === a && spans[k][1] === b && (!pick || spans[k][3] < pick[3])) + pick = spans[k]; + const rep = pick ? pick[2] : '[REDACTED]'; + out += s.slice(at, a) + rep + (keepLines ? '\n'.repeat(Math.max(0, breaks(s.slice(a, b)) - breaks(rep))) : ''); + at = b; + i = j; + } + return out + s.slice(at); +} +function replaceUnion(s, keepLines) { + let out = replaceOnce(s, keepLines); + for (let i = 0; i < 2 && out !== s; i++) { + const again = replaceOnce(out, keepLines); + if (again === out) + break; + out = again; + } + return out; +} +function maskOnce(s, mask, keepLines) { + const spans = allSpans(s, BROAD_RULES); const word = (i) => /[A-Za-z0-9_]/.test(s[i] ?? ''); - readAt(s, gluePoints(s), spans); - readAt(s, [...new Set(spans.flat())].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), spans); + readAt(s, [...new Set(spans.flatMap(([a, b]) => [a, b]))].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), BROAD_RULES, spans); const masks = []; if (mask) for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) masks.push([at, at + mask.length]); - const kept = outside(merged(spans), masks); + const kept = outside(merged(spans.map(([a, b]) => [a, b])), masks); if (!kept.length) return s; let out = ''; @@ -385,7 +431,7 @@ function merged(spans) { } return out; } -function readAt(s, cuts, spans) { +function readAt(s, cuts, rules, spans) { if (!cuts.length) return; const markChar = freeMark(s); @@ -393,10 +439,9 @@ function readAt(s, cuts, spans) { for (const [i, c] of cuts.entries()) { const piece = s.slice(c, cuts[i + 1] ?? s.length); const more = []; - for (const [re, rep] of BROAD_RULES) - spansOf(piece, re, rep, more); - for (const [a, b] of more) - spans.push([a + c, b + c]); + rules.forEach(([re, rep], k) => spansOf(piece, re, rep, k, more)); + for (const [a, b, r, k] of more) + spans.push([a + c, b + c, r, k]); } return; } @@ -420,10 +465,9 @@ function readAt(s, cuts, spans) { return p - lo; }; const more = []; - for (const [re, rep] of BROAD_RULES) - spansOf(marked, re, rep, more); - for (const [a, b] of more) - spans.push([back(a), back(b)]); + rules.forEach(([re, rep], k) => spansOf(marked, re, rep, k, more)); + for (const [a, b, r, k] of more) + spans.push([back(a), back(b), r.split(markChar).join(''), k]); } function outside(spans, regions) { const out = []; @@ -466,8 +510,7 @@ function redactString(s, o = {}) { } function redactOrThrow(s, o) { const put = o.keepLines ? applyKeepingLines : apply; - const secrets = (t) => PATTERNS.reduce((y, [re, rep]) => put(y, re, rep), t); - let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : glued(secrets(s), secrets); + let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : replaceUnion(s, o.keepLines === true); if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); return out; diff --git a/src/dashboard.ts b/src/dashboard.ts index e09cbf481..af7be4607 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -884,7 +884,6 @@ const journalctl: JournalReader = (unit, lines) => { return { status: r.status, stdout: r.stdout ?? '', stderr: `${r.stderr ?? ''}${r.error ? r.error.message : ''}` }; }; -/** A message cut to 2,000 characters, redacted first: a cut made before would leave a long secret's body unfound (#889). */ /** How much of one log record is shown, and how much of it is read (redacted) to show that (#889 round 4). */ const RECORD_SHOWN = 2000; const RECORD_READ = RECORD_SHOWN + 4096; diff --git a/src/events/redact.ts b/src/events/redact.ts index e2e780bcb..2d362fd56 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -61,6 +61,42 @@ const DASHBOARD_KEY_BODY = '[A-Za-z0-9_-]{32,}'; /** Whether `key` is a dashboard key of the current shape (prefixed). */ export const isDashboardKey = (key: string): boolean => new RegExp(`^${DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}$`).test(key); +const KEY_LINE = /-----(BEGIN|END) [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/g; +/** + * `m` (the text up to its last END line) with everything up to its last END that no BEGIN opened redacted; as it was + * when each END has its BEGIN. A BEGIN opens a key up to the next END, as the BEGIN rule reads it (#889 round 6: the + * rules read the text as it was, so a closed key's END is still there, and it is not an orphan). One scan: linear. + */ +function orphanEnd(m: string): string { + let open = false; + let cut = -1; + KEY_LINE.lastIndex = 0; + for (let k = KEY_LINE.exec(m); k; k = KEY_LINE.exec(m)) { + if (k[1] === 'BEGIN') open = true; + else if (open) open = false; + else cut = k.index + k[0].length; + } + return cut === -1 ? m : `[REDACTED PRIVATE KEY]${m.slice(cut)}`; +} + +/** + * `m` (a run of three or more dotted parts after `pre`) with a JWT in it redacted, from its `eyJ` to the run's end: the first part + * holding `eyJ` and 8 more characters, followed by a part of 8 or more (the payload) and one more dot. Any part may be + * the header (#889 round 6: `name.eyJ...`, where the run's first part is a name, was found by main's rule from the + * dot); within a part, the header may be glued after anything (round 5). As it was when no part is a header. One walk + * over the parts: linear. + */ +function jwtIn(m: string, pre: string): string { + const parts = m.slice(pre.length).split('.'); + let before = pre; + for (let i = 0; i + 2 < parts.length; i++) { + const at = parts[i]!.indexOf('eyJ'); + if (at !== -1 && parts[i]!.length - at >= 11 && parts[i + 1]!.length >= 8) return `${before}${parts[i]!.slice(0, at)}[REDACTED JWT]`; + before += `${parts[i]}.`; + } + return m; +} + // Each pattern with what kind of secret it finds (secretKinds names the kind, never the value). const PATTERNS: [RegExp, Replacement, string][] = [ // A dashboard key, wherever its prefix is (after a `-` or a letter too: `key-wld_...`): a match runs to the end of its @@ -85,14 +121,10 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // Read from the start of the header's run of base64url characters, with no bound on its length (an `x5c` chain makes // a header several KB, #889 round 5); a function finds `eyJ` in that run, so a token glued after a letter, a digit or // `-` is found from its `eyJ` on. Each start is a run's start: linear. - [ - /(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{11,})\.([A-Za-z0-9_-]{8,})\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, - (m, pre, head) => { - const at = head!.indexOf('eyJ'); - return at !== -1 && head!.length - at >= 11 ? `${pre}${head!.slice(0, at)}[REDACTED JWT]` : m; - }, - 'JWT', - ], + // Each of the first two parts is read whole, with no way back into it (a lookahead takes the run, a backreference + // steps over it): a run of millions of characters with no dot after it is passed in one step, not given back a + // character at a time until V8's backtracking stack overflows (#889 round 6). jwtIn checks the parts' lengths. + [/(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]+))\2\.(?=([A-Za-z0-9_-]+))\3\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, (m, pre) => jwtIn(m, pre!), 'JWT'], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], // Slack's token-rotation and refresh tokens (xoxe-..., xoxe.xoxb-...), before the // other Slack tokens (one is inside it); then its app-level token (xapp-, Socket Mode). @@ -110,7 +142,7 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // no BEGIN before it (a key whose start is above a window), from the start of the text to that END line. Each BEGIN // is read once (its match runs to its END or to the end), and the second rule has one start: linear. [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], - [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, '[REDACTED PRIVATE KEY]', 'private key'], + [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, orphanEnd, 'private key'], // The scheme is case-insensitive (RFC 9110): any case (#889). `bearer` is a word, so after a boundary. [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], // HTTP Basic credentials (base64 of user:password; git's extraheader, src/dashboard-auth.ts's client id and secret), @@ -296,30 +328,6 @@ function freeMark(s: string): string | null { return null; } -/** - * The second pass (#889): `f` again on `s` with a boundary at each glue point, so a token glued to a name, to an ANSI - * escape or to a JSON escape is found from its start. The boundary is a mark no pattern takes in, inserted and taken - * out again, so the text is otherwise kept as it was. When `s` holds every mark character, each piece after a glue - * point is redacted on its own instead. The first pass ran on the text as it was, so nothing it found is lost. - */ -function glued(s: string, f: (part: string) => string): string { - const cuts = gluePoints(s); - if (!cuts.length) return s; - const mark = freeMark(s); - if (mark === null) { - let out = s.slice(0, cuts[0]); - for (const [i, c] of cuts.entries()) out += f(s.slice(c, cuts[i + 1] ?? s.length)); - return out; - } - let marked = ''; - let from = 0; - for (const c of cuts) { - marked += s.slice(from, c) + mark; - from = c; - } - return f(marked + s.slice(from)).split(mark).join(''); -} - /** `$1`, `$&` and `$$` in a replacement string, as String.prototype.replace reads them. */ const expand = (rep: string, m: string, groups: unknown[]) => rep.replace(/\$(\d|\$|&)/g, (_, k: string) => (k === '$' ? '$' : k === '&' ? m : String(groups[Number(k) - 1] ?? ''))); const breaks = (s: string) => s.split('\n').length - 1; @@ -378,12 +386,23 @@ function outsideMachineBlocks(s: string, f: (part: string) => string): string { return out + f(s.slice(from)); } -/** The kinds of secret `s` looks like it holds (sorted, each once): names only, never a matched value. */ +/** The kind secretKinds names for a text it can't scan (#889 round 6): a caller that refuses a secret refuses it too. */ +export const UNSCANNABLE_KIND = 'text too large to scan'; + +/** + * The kinds of secret `s` looks like it holds (sorted, each once): names only, never a matched value. A text with a + * run too long to scan (V8's RangeError, as in redactString) fails closed: it is named UNSCANNABLE_KIND. + */ export function secretKinds(s: string): string[] { const kinds = new Set(); - for (const [re, rep, kind] of PATTERNS) { - re.lastIndex = 0; - if (apply(s, re, rep) !== s) kinds.add(kind); + try { + for (const [re, rep, kind] of PATTERNS) { + re.lastIndex = 0; + if (apply(s, re, rep) !== s) kinds.add(kind); + } + } catch (e) { + if (e instanceof RangeError) return [UNSCANNABLE_KIND]; + throw e; } return [...kinds].sort(); } @@ -396,12 +415,16 @@ const globalOf = (re: RegExp) => { return g; }; +/** What a rule found: [start, end), what its replacement puts there, and the rule's place in its list. */ +type Span = [number, number, string, number]; + /** * Where `re` finds a secret in `t`, as [start, end) spans: each match's part its replacement takes out (what the match * and the replacement don't share at either end), so the scheme, the key's name or the separator a rule keeps is not - * in the span. A match its replacement leaves as it was (a function that judged it no secret) gives none. + * in the span; with what the replacement puts there instead. A match its replacement leaves as it was (a function + * that judged it no secret) gives none. */ -function spansOf(t: string, re: RegExp, rep: Replacement, out: [number, number][]) { +function spansOf(t: string, re: RegExp, rep: Replacement, rank: number, out: Span[]) { const g = globalOf(re); g.lastIndex = 0; for (let m = g.exec(t); m; m = g.exec(t)) { @@ -417,12 +440,65 @@ function spansOf(t: string, re: RegExp, rep: Replacement, out: [number, number][ while (a < whole.length && a < r.length && whole[a] === r[a]) a++; let b = 0; while (b < whole.length - a && b < r.length - a && whole[whole.length - 1 - b] === r[r.length - 1 - b]) b++; - if (whole.length - b > a) out.push([m.index! + a, m.index! + whole.length - b]); + if (whole.length - b > a) out.push([m.index! + a, m.index! + whole.length - b, r.slice(a, r.length - b), rank]); } } +/** Every rule of the default mode: the shapes, each with its own marker. */ +const SHAPE_RULES: [RegExp, Replacement][] = PATTERNS.map(([re, rep]): [RegExp, Replacement] => [re, rep]); /** Every rule of the broad mode: the shapes and the broad rules, each shown with the default mask. */ -const BROAD_RULES: [RegExp, Replacement][] = [...PATTERNS.map(([re, rep]): [RegExp, Replacement] => [re, rep]), ...BROAD.map(([re, rep]): [RegExp, Replacement] => [re, rep('[REDACTED]')])]; +const BROAD_RULES: [RegExp, Replacement][] = [...SHAPE_RULES, ...BROAD.map(([re, rep]): [RegExp, Replacement] => [re, rep('[REDACTED]')])]; + +/** What every rule of `rules` finds in `s`: on the text as it was, and again with a boundary at each glue point. */ +function allSpans(s: string, rules: readonly [RegExp, Replacement][]): Span[] { + const spans: Span[] = []; + rules.forEach(([re, rep], i) => spansOf(s, re, rep, i, spans)); + readAt(s, gluePoints(s), rules, spans); + return spans; +} + +/** + * The default mode (#889 round 6, the round-3+4 delta check's regression finding): every shape runs on the text as it + * was (and from each glue point), and each run of overlapping spans is replaced once. Order-free by design: no rule's + * match can take in the text another rule starts from (`wld_-redis://u:@`, `.Bearer `), so each + * shape hides at least what it hid alone; test/redact-default-union.test.ts holds this against main's rules on + * composed input. A span that is the whole run keeps its rule's marker (the first rule's, when several are); a run of + * spans that only overlap becomes `[REDACTED]`. One pass of replaceUnion. + */ +function replaceOnce(s: string, keepLines: boolean): string { + const spans = allSpans(s, SHAPE_RULES); + if (!spans.length) return s; + spans.sort((x, y) => x[0] - y[0] || y[1] - x[1] || x[3] - y[3]); + let out = ''; + let at = 0; + for (let i = 0; i < spans.length; ) { + const a = spans[i]![0]; + let b = spans[i]![1]; + let j = i + 1; + for (; j < spans.length && spans[j]![0] < b; j++) b = Math.max(b, spans[j]![1]); + let pick: Span | undefined; + for (let k = i; k < j; k++) if (spans[k]![0] === a && spans[k]![1] === b && (!pick || spans[k]![3] < pick[3])) pick = spans[k]; + const rep = pick ? pick[2] : '[REDACTED]'; + out += s.slice(at, a) + rep + (keepLines ? '\n'.repeat(Math.max(0, breaks(s.slice(a, b)) - breaks(rep))) : ''); + at = b; + i = j; + } + return out + s.slice(at); +} + +/** + * The default mode: replaced once, then again on what that gave, until nothing changes (at most 3 times). A marker can + * make a boundary a later rule needs, as main's rules, applied in turn, had. + */ +function replaceUnion(s: string, keepLines: boolean): string { + let out = replaceOnce(s, keepLines); + for (let i = 0; i < 2 && out !== s; i++) { + const again = replaceOnce(out, keepLines); + if (again === out) break; + out = again; + } + return out; +} /** * The broad mode (#889 round 3, the final check's regression finding): every rule runs on the text as it was, and the @@ -432,18 +508,17 @@ const BROAD_RULES: [RegExp, Replacement][] = [...PATTERNS.map(([re, rep]): [RegE * are found the same way, on the text with a mark at each, mapped back. One pass of maskUnion. */ function maskOnce(s: string, mask: string, keepLines: boolean): string { - const spans: [number, number][] = []; - for (const [re, rep] of BROAD_RULES) spansOf(s, re, rep, spans); - // Two more reads, each with boundaries marked in: the glue points; and where a span found begins or ends between two - // word characters (a module's own rules, applied in turn, saw a boundary where an earlier rule's replacement ended: - // `xoxa-...[here]_db_password=...`). Apart, so a boundary of one kind can't break a match the other needs. + // Every rule, on the text as it was and from each glue point (allSpans); then one more read with a boundary marked in + // where a span found begins or ends between two word characters (a module's own rules, applied in turn, saw a + // boundary where an earlier rule's replacement ended: `xoxa-...[here]_db_password=...`). Apart from the glue points, + // so a boundary of one kind can't break a match the other needs. + const spans = allSpans(s, BROAD_RULES); const word = (i: number) => /[A-Za-z0-9_]/.test(s[i] ?? ''); - readAt(s, gluePoints(s), spans); - readAt(s, [...new Set(spans.flat())].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), spans); + readAt(s, [...new Set(spans.flatMap(([a, b]) => [a, b]))].filter((c) => c > 0 && c < s.length && word(c - 1) && word(c)).sort((a, b) => a - b), BROAD_RULES, spans); // What is already a mask stays as it is: a rule that takes in a mask word (`token=[REDACTED]`) adds nothing. const masks: [number, number][] = []; if (mask) for (let at = s.indexOf(mask); at !== -1; at = s.indexOf(mask, at + mask.length)) masks.push([at, at + mask.length]); - const kept = outside(merged(spans), masks); + const kept = outside(merged(spans.map(([a, b]): [number, number] => [a, b])), masks); if (!kept.length) return s; let out = ''; let at = 0; @@ -466,17 +541,20 @@ function merged(spans: [number, number][]): [number, number][] { return out; } -/** Every rule on `s` with a boundary marked in at each of `cuts` (sorted); the spans found, mapped back, go in `spans`. */ -function readAt(s: string, cuts: number[], spans: [number, number][]) { +/** + * Every one of `rules` on `s` with a boundary marked in at each of `cuts` (sorted), so a token glued to what is before + * it is read from its start; the spans found, mapped back, go in `spans`. The boundary is a character no rule takes in + * (a private-use one the text doesn't hold); when the text holds them all, each piece after a cut is read on its own. + */ +function readAt(s: string, cuts: number[], rules: readonly [RegExp, Replacement][], spans: Span[]) { if (!cuts.length) return; const markChar = freeMark(s); if (markChar === null) { - // Every mark character is in the text: each piece after a cut is read on its own. for (const [i, c] of cuts.entries()) { const piece = s.slice(c, cuts[i + 1] ?? s.length); - const more: [number, number][] = []; - for (const [re, rep] of BROAD_RULES) spansOf(piece, re, rep, more); - for (const [a, b] of more) spans.push([a + c, b + c]); + const more: Span[] = []; + rules.forEach(([re, rep], k) => spansOf(piece, re, rep, k, more)); + for (const [a, b, r, k] of more) spans.push([a + c, b + c, r, k]); } return; } @@ -498,9 +576,9 @@ function readAt(s: string, cuts: number[], spans: [number, number][]) { } return p - lo; }; - const more: [number, number][] = []; - for (const [re, rep] of BROAD_RULES) spansOf(marked, re, rep, more); - for (const [a, b] of more) spans.push([back(a), back(b)]); + const more: Span[] = []; + rules.forEach(([re, rep], k) => spansOf(marked, re, rep, k, more)); + for (const [a, b, r, k] of more) spans.push([back(a), back(b), r.split(markChar).join(''), k]); } /** @@ -553,10 +631,10 @@ export function redactString(s: string, o: RedactTextOptions = {}): string { function redactOrThrow(s: string, o: RedactTextOptions): string { const put = o.keepLines ? applyKeepingLines : apply; - // The broad mode masks the union of every rule's spans (maskUnion); the default mode applies the shapes in turn, - // each with its own marker, then again from each glue point (glued). - const secrets = (t: string) => PATTERNS.reduce((y, [re, rep]) => put(y, re, rep), t); - let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : glued(secrets(s), secrets); + // Both modes read every rule on the text as it was, so no rule's match hides another's: the broad mode masks the + // union of what the shapes and the broad rules find (maskUnion); the default mode replaces each run of what the + // shapes find, with its rule's marker (replaceUnion). + let out = o.broad ? maskUnion(s, o.mask ?? '[REDACTED]', o.keepLines === true) : replaceUnion(s, o.keepLines === true); if (o.identifiers) out = outsideMachineBlocks(out, (part) => IDENTIFIERS.reduce((t, [re, rep]) => put(t, re, rep), part)); return out; } diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 09bb34f79..9072e7421 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -96,6 +96,9 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'test/redact-records-keys.test.ts', table: 'CUT_SHORT', kind: 'never-shrink' }, { path: 'test/token-standins.ts', table: 'SHAPES', kind: 'never-shrink' }, { path: 'test/redaction-one-module.test.ts', table: 'BAD_SOURCES', kind: 'never-shrink' }, + // Main's default rules, the baseline the default mode must hide at least as much as (#889 round 6). + { path: 'test/redact-default-union.test.ts', table: 'MAIN_PATTERNS', kind: 'never-shrink' }, + { path: 'test/redact-default-union.test.ts', table: 'CREDENTIAL_WORDS', kind: 'never-shrink' }, // The guards' own tables (their files are guard files too; the table names what was taken out). { path: 'src/review.ts', table: 'SELF_MODIFICATION_PARTS', kind: 'never-shrink' }, { path: 'src/review.ts', table: 'LOCKED_CATEGORIES', kind: 'never-shrink' }, diff --git a/test/redact-default-union.test.ts b/test/redact-default-union.test.ts new file mode 100644 index 000000000..e7cb533a3 --- /dev/null +++ b/test/redact-default-union.test.ts @@ -0,0 +1,177 @@ +// #889 round 6 (the round-3+4 delta check's regression finding): the default mode (redactString with no options: the +// event log, comments, the digest, the dashboard) hides at least what main's default mode hid. Each rule alone was the +// same or broader, but the rules applied one after another, and one rule's match ate the next rule's anchor +// (`wld_-redis://u:@`, `.Bearer `, `xoxb-...-lin_api_`). Main's PATTERNS are kept here +// verbatim, as a fixture, from origin/main d2a549f4, applied as main applied them; on every pair of full-length shapes +// joined by each separator, and on generated fragments, every secret main hid stays hidden. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { redactString } from '../src/events/redact.js'; +import { leaks, run } from './token-standins.js'; + +type Replacement = string | ((match: string, ...groups: string[]) => string); +// Main's helpers for its JSON-key rule, verbatim. +const credentialShaped = (v: string) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); +const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'jwt', 'bearer', 'credential', 'credentials', 'authorization']); +const CREDENTIAL_ENDINGS = ['token', 'secret', 'password', 'passwd', 'accesskey']; +const credentialWord = (w: string) => CREDENTIAL_WORDS.has(w) || CREDENTIAL_ENDINGS.some((e) => w.endsWith(e)); +const KEY_QUALIFIERS = new Set(['api', 'access', 'secret', 'private', 'signing', 'encryption', 'master', 'client', 'app', 'auth', 'account', 'service', 'license', 'deploy', 'admin', 'webhook', 'session', 'ssh', 'hmac', 'consumer']); +function credentialKey(k: string): boolean { + const words = k + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Z])([A-Z][a-z])/g, '$1 $2') + .toLowerCase() + .split(/[ _-]+/) + .filter(Boolean); + const last = words.at(-1) ?? ''; + if (words.some(credentialWord) || last === 'auth') return true; + if (last === 'key') return words.length === 1 || KEY_QUALIFIERS.has(words.at(-2)!); + return last.endsWith('key') && KEY_QUALIFIERS.has(last.slice(0, -3)); +} + +/** Main's PATTERNS (src/events/redact.ts at d2a549f4), verbatim. */ +const MAIN_PATTERNS: [RegExp, Replacement][] = [ + [/\b(gh[pousr]_[A-Za-z0-9]{30,})\b/g, 'gh*_[REDACTED]'], + [/\bgithub_pat_[A-Za-z0-9_]{40,}\b/g, 'github_pat_[REDACTED]'], + [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]'], + [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v!) ? `${pre}${k}_[REDACTED]` : m)], + [/(^|[^A-Za-z0-9_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, '$1AIza[REDACTED]'], + [/(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*/g, '$1[REDACTED JWT]'], + [/\b(AKIA|ASIA)[A-Z0-9]{16}\b/g, 'AKIA[REDACTED]'], + [/\bxoxe[.-][A-Za-z0-9.-]{10,}\b/g, 'xoxe-[REDACTED]'], + [/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g, 'xox-[REDACTED]'], + [/\bxapp-[A-Za-z0-9-]{10,}\b/g, 'xapp-[REDACTED]'], + [/\blin_(?:api|oauth)_[A-Za-z0-9]{20,}\b/g, 'lin_[REDACTED]'], + [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/[REDACTED]'], + [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '[REDACTED PRIVATE KEY]'], + [/\b(Bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/g, '$1 [REDACTED]'], + [/\b([a-z][a-z0-9+.-]*:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@'], + [/\b0x[0-9a-fA-F]{64}\b/g, '0x[REDACTED-32-BYTES]'], + [ + /(\\?")([A-Za-z0-9_-]+)\1(\s*:\s*)\1([A-Za-z0-9_\-.+/=~]{16,})\1/g, + (m, q, k, sep, v) => (credentialKey(k!) && credentialShaped(v!) ? `${q}${k}${q}${sep}${q}[REDACTED]${q}` : m), + ], +]; +/** Main's default mode: its PATTERNS one after another. */ +const mainDefault = (s: string) => MAIN_PATTERNS.reduce((t, [re, rep]) => (typeof rep === 'string' ? t.replace(re, rep) : t.replace(re, rep)), s); + +const KEY_BEGIN = ['-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const KEY_END = ['-----END', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const hex = (n: number) => 'c0ffee5ba11d0d9e'.repeat(Math.ceil(n / 16)).slice(0, n); + +/** Full-length shapes, each with its own secret (no two share a 12-character piece). Prefixes put together at run time. */ +const SHAPES: [string, string, string][] = (() => { + const s = (n: number, seed: string) => run(n, seed); + const jwtSig = s(43, 'Jw7SgN4tR9xQ'); + return [ + ['dashboard key', `${'wld'}_${s(32, 'Dk3Yq8Wm2Lp5')}`, s(32, 'Dk3Yq8Wm2Lp5')], + ['GitHub token', `${'ghp'}_${s(36, 'Gh5Tk9Rz2Xw4')}`, s(36, 'Gh5Tk9Rz2Xw4')], + ['fine-grained GitHub token', `${'github'}_pat_${s(60, 'Fg8Pa3Tn6Qe1')}`, s(60, 'Fg8Pa3Tn6Qe1')], + ['Anthropic key', `${'sk'}-ant-api03-${s(40, 'An7Ck2Ey5Hb9')}`, s(40, 'An7Ck2Ey5Hb9')], + ['plain sk- key', `${'sk'}-${s(40, 'Sp4Kx8Lm1Vd6')}`, s(40, 'Sp4Kx8Lm1Vd6')], + ['Stripe key', `${'sk'}_live_${s(24, 'St2Rp7Ik4Yz9')}`, s(24, 'St2Rp7Ik4Yz9')], + ['Slack bot token', `${'xoxb'}-123456789012-${s(24, 'Sl6Bt1Kq8Wn3')}`, s(24, 'Sl6Bt1Kq8Wn3')], + ['Slack app token', `${'xapp'}-1-${s(30, 'Xa9Pp2Tk5Mr7')}`, s(30, 'Xa9Pp2Tk5Mr7')], + ['Linear key', `${'lin'}_api_${s(40, 'Ln3Ar8Pk6Qc2')}`, s(40, 'Ln3Ar8Pk6Qc2')], + ['JWT', `${'eyJ'}${s(20, 'hDr1Bq5Zt8')}.${'eyJ'}${s(24, 'pLd4Nx7Jv2')}.${jwtSig}`, jwtSig], + ['AWS key', `${'AKIA'}${'Q7RZ4KX2M9TBH3WD'}`, 'Q7RZ4KX2M9TBH3WD'], + ['Google key', `${'AIza'}${s(35, 'Go4Gk1Ey9Ap3')}`, s(35, 'Go4Gk1Ey9Ap3')], + ['Tailscale key', `${'tskey'}-auth-${s(30, 'Ts2Ak7Qz4Wf8')}`, s(30, 'Ts2Ak7Qz4Wf8')], + ['bearer token', `Bearer ${s(36, 'Br5Tk2Nv8Zx3')}`, s(36, 'Br5Tk2Nv8Zx3')], + ['URL password', `postgres://app:${s(20, 'Pw6Db3Hq9Lk1')}@db.example.com`, s(20, 'Pw6Db3Hq9Lk1')], + ['32-byte hex key', `0x${hex(64)}`, hex(64)], + ['JSON value', `{"api_key":"${s(28, 'Js8Vk3Ap5Qy2')}"}`, s(28, 'Js8Vk3Ap5Qy2')], + ['private key', `${KEY_BEGIN}\n${s(64, 'Pk1Bd7Ry4Ms9')}\n${KEY_END}`, s(64, 'Pk1Bd7Ry4Ms9')], + ]; +})(); + +/** Separators a log line, a URL, JSON or an ANSI-coloured terminal puts between two values. */ +const SEPARATORS = ['', ' ', '-', '.', '+', '_', ':', '=', ',', '/', '"', '\n', '\u001b[1m', '\\n', 'Authorization: ', 'BEARER ']; + +/** The secrets main hid in `s` that `out` shows. */ +function narrowed(s: string, out: string, secrets: string[]): string[] { + const old = mainDefault(s); + return secrets.filter((x) => s.includes(x) && !leaks(old, x) && leaks(out, x)); +} + +test('on every pair of full-length shapes joined by each separator, the default mode hides everything main hid', () => { + const missed: string[] = []; + for (const [aName, a, aSecret] of SHAPES) { + for (const [bName, b, bSecret] of SHAPES) { + for (const sep of SEPARATORS) { + for (const s of [`${a}${sep}${b}`, `x ${a}${sep}${b} y`]) { + const shown = narrowed(s, redactString(s), [aSecret, bSecret]); + if (shown.length && missed.length < 40) missed.push(`${aName} ${JSON.stringify(sep)} ${bName}: ${JSON.stringify(s).slice(0, 120)}`); + } + } + } + } + assert.deepEqual(missed, []); +}); + +test("the delta check's cases: one rule's match no longer eats the next rule's anchor", () => { + const pw = run(20, 'Zq4Pw8Ld2Mx6'); + const tok = run(30, 'Bt3Rq9Wv6Kc1'); + const lin = run(40, 'Li7Nk2Ey5Rb8'); + const jwt = `${'eyJ'}${run(20, 'hDr1Bq5Zt8')}.${'eyJ'}${run(24, 'pLd4Nx7Jv2')}.${run(43, 'Jw7SgN4tR9xQ')}`; + const cases: [string, string][] = [ + [`${'wld'}_${run(32, 'Dk3Yq8Wm2Lp5')}-redis://app:${pw}@db.example.com`, pw], + [`${jwt}.postgres://app:${pw}@db.example.com`, pw], + [`${jwt}.Bearer ${tok}`, tok], + [`${'xoxb'}-123456789012-${run(24, 'Sl6Bt1Kq8Wn3')}-${'lin'}_api_${lin}`, lin], + [`${'xapp'}-1-${run(30, 'Xa9Pp2Tk5Mr7')}-${'lin'}_api_${lin}`, lin], + [`Authorization: BEARER ${tok}+postgres://app:${pw}@db.example.com`, pw], + [`${'tskey'}-auth-${run(30, 'Ts2Ak7Qz4Wf8')}-Bearer ${tok}`, tok], + [`"a\\n${'sk'}-ant-oat01-${run(40, 'An7Ck2Ey5Hb9')}-postgres://app:${pw}@db.example.com"`, pw], + // A JWT after a dotted name whose first part is long: main's rule found it from the dot. + [`${run(16, 'Nm5Qa8Zr3Kd1')}-api_key.${jwt}`, run(43, 'Jw7SgN4tR9xQ')], + [`see ${run(13, 'Wq2Lp6Xn9Bc4')}.${'eyJ'}${run(14, 'Hd3Rz7Kq1Mv5')}.${run(14, 'Py8Lx2Tw6Nb3')}.${run(16, 'Sg4Qe9Ja2Wz7')}`, run(16, 'Sg4Qe9Ja2Wz7')], + // The round-5 delta check's JWT cases: a dotted segment of 11 or more characters before the token. + [`token abcdefghijk.${jwt} end`, run(43, 'Jw7SgN4tR9xQ')], + [`GET api.example-service.${jwt} 200`, run(43, 'Jw7SgN4tR9xQ')], + [`Cookie: s=sess_0123456789.${jwt}`, run(43, 'Jw7SgN4tR9xQ')], + [`https://auth.example.com/cb/abcdefghijklm.${jwt}`, run(43, 'Jw7SgN4tR9xQ')], + [`token abcdefghijk.${'eyJ'}${run(600, 'hDr1Bq5Zt8')}.${'eyJ'}${run(24, 'pLd4Nx7Jv2')}.${run(43, 'Jw7SgN4tR9xQ')} end`, run(43, 'Jw7SgN4tR9xQ')], + ]; + const shown = cases.filter(([s, secret]) => !leaks(mainDefault(s), secret) && leaks(redactString(s), secret)).map(([s]) => s.slice(0, 60)); + assert.deepEqual(shown, []); +}); + +test('on generated fragments and whole tokens, the default mode hides everything main hid', () => { + const SECRETS = ['Q1w2E3r4', 'Z9y8X7w6V5u4', 'Mn3Bv4Cx5Zl6Kj7H', 'pL0oK9iJ8uH7yG6tF5rD4eS3']; + const tail = SECRETS[3]!; + const COMMON = ['.', 'bearer ', 'Bearer ', 'BEARER ', 'https://', 'git+ssh://', 'redis://', '@', ':', '=', ' ', ' ', '-', '+', '_', 'token', 'Password', 'api_key', '"', "'", '/', '\n', KEY_BEGIN, KEY_END, 'x', ',', '\u001b[1m', '\\n', '7']; + // Prefixes alone (the delta check's `frag` set), and whole tokens ending in a secret (its `real` set). + const SETS: [string, string[]][] = [ + ['fragments', ['ghp_', 'gho_', 'github_pat_', 'sk-ant-', 'sk-', 'xoxb-', 'xoxp-', 'xapp-', 'lin_api_', 'eyJ', 'AKIA', 'AIza', 'wld_', 'tskey-', '0x', ...COMMON]], + [ + 'whole tokens', + [`${'ghp'}_${tail}Mn3Bv4Cx5Zl6`, `${'github'}_pat_11${tail}_Mn3Bv4Cx5Zl6Kj7H`, `${'sk'}-ant-api03-${tail}`, `${'xoxb'}-123456789012-${tail}`, `${'eyJ'}hbGciOiJIUzI1NiJ9.${'eyJ'}zdWIiOiIxIn0.${tail}`, `${'AKIA'}Q7RZ4KX2M9TBQ7RZ`, `${'tskey'}-auth-k${tail}`, `${'wld'}_${tail}Mn3Bv4Cx5Z`, `${'lin'}_api_${tail}`, ...COMMON], + ], + ]; + const missed: string[] = []; + for (const [set, PIECES] of SETS) { + let seed = 2463534242; + const rnd = (k: number) => { + seed ^= seed << 13; + seed >>>= 0; + seed ^= seed >>> 17; + seed ^= seed << 5; + seed >>>= 0; + return seed % k; + }; + for (let i = 0; i < 40_000; i++) { + let s = ''; + for (let j = 2 + rnd(6); j > 0; j--) s += rnd(3) === 0 ? SECRETS[rnd(SECRETS.length)] : PIECES[rnd(PIECES.length)]; + const present = SECRETS.filter((x) => s.includes(x)); + if (!present.length) continue; + const old = mainDefault(s); + const out = redactString(s); + // Occurrences, as the delta check counted them: no secret is shown more often than main showed it. + const occ = (t: string, x: string) => t.split(x).length - 1; + if (present.some((x) => occ(out, x) > occ(old, x)) && missed.length < 20) missed.push(`${set}: ${JSON.stringify(s)}`); + } + } + assert.deepEqual(missed, []); +}); diff --git a/test/redact-unscannable.test.ts b/test/redact-unscannable.test.ts new file mode 100644 index 000000000..15007caf2 --- /dev/null +++ b/test/redact-unscannable.test.ts @@ -0,0 +1,45 @@ +// #889 round 6 (the round-5 delta check): a text with a run of millions of characters. +// - secretKinds fails closed: it names the text as too large to scan, so an attachment is refused (409) until the +// person attaches it anyway, instead of throwing V8's RangeError. +// - A plain base64url run (no prefix, no dot) is left as it was, as on main: the JWT rule passes over it in one step. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as redactModule from '../src/events/redact.js'; +import { redactString, secretKinds } from '../src/events/redact.js'; +import { AttachmentError, ATTACHMENT_DEFAULT_CAPS, storeUpload } from '../src/chat-attachments.js'; + +const SIX_MB = 6 * 1024 * 1024; +const huge = `${'ghp'}_${'a'.repeat(SIX_MB)}`; + +test('secretKinds names a text it cannot scan instead of throwing', () => { + let kinds: string[] = []; + assert.doesNotThrow(() => { + kinds = secretKinds(huge); + }); + assert.equal(typeof redactModule.UNSCANNABLE_KIND, 'string'); + assert.deepEqual(kinds, [redactModule.UNSCANNABLE_KIND]); + // An ordinary text still gets its kinds. + assert.deepEqual(secretKinds(`x ${'ghp'}_${'Zq9Xw8Vu7Ts6Rq5Po4Nm3Lk2Ji1Hg0FeDcBa'} y`), ['GitHub token']); +}); + +test('an attachment too large to scan is refused (409) until attached anyway', async () => { + const stateDir = mkdtempSync(join(tmpdir(), 'attach-huge-')); + async function* once() { + yield Buffer.from(huge); + } + const put = (anyway: boolean) => storeUpload({ stateDir, name: 'big.txt', body: once(), caps: { ...ATTACHMENT_DEFAULT_CAPS }, by: 'example-collaborator', gitleaks: null, anyway }); + await assert.rejects(put(false), (e: unknown) => e instanceof AttachmentError && e.status === 409 && e.secretKinds.join() === redactModule.UNSCANNABLE_KIND); + const m = await put(true); + assert.deepEqual(m.secretKinds, [redactModule.UNSCANNABLE_KIND]); +}); + +test('a plain base64url run of 6 MB is left as it was (default, broad) and names no kind', () => { + const plain = 'aB3-_x'.repeat(SIX_MB / 6); + assert.equal(redactString(plain), plain); + assert.equal(redactString(plain, { broad: true }), plain); + assert.deepEqual(secretKinds(plain), []); +}); From 2195272177c07cd2d6a9322a466742f2b0e6ef86 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 10:07:33 -0500 Subject: [PATCH 8/9] redaction: a declined match hides nothing it stepped over, JWT rule declines nothing, passes fail closed (#889) Round 7, on the round-6 second review and the round-6 delta (#921): - The JWT rule is a plain replacement again: it starts only at a part holding `eyJ` (a lookahead looks in the part, so each part is read once), steps over the header and payload whole (no stack overflow on a huge run), and takes further dotted parts, empty ones too. An unsigned JWT (`alg: none`) or one with a `+`/`/`-led signature after a dotted name (`a..`) was shown and secretKinds missed it: the match stopped before the empty part, its function declined it, and the 64-character restart stepped past the JWT's own start. - By design for every rule: a match a rule's function declines restarts the search at its next character (spansOf), so no start inside it is skipped. test/redact-declined.test.ts holds every function-backed rule to a reference that tries every start, fails when a new one isn't listed, and times 1 MB of declined text. - The passes fail closed (fixedPoint): each mode passes over its own output until nothing changes, at most MAX_PASSES (8) times; a text still changing becomes the placeholder. Before, each stopped after 3 passes. - The Basic value ends where its run and padding end, whatever follows: broad mode showed `Basic ==` that default found only in a later pass. - test/redact-default-union.test.ts adds unsigned and `+`-led JWTs and the `x.` and `a.b.` separators; its KEY_QUALIFIERS and CREDENTIAL_ENDINGS are registered never-shrink. Tests (fail without): test/redact-declined.test.ts and test/redact-fixed-point.test.ts, 4 of 6 failing at 1eef3625 (the other two are a timing guard and an idempotence guard). Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 2 +- scripts/machine/worklane-update.cjs | 51 +++++------ src/events/redact.ts | 95 +++++++++++---------- src/merge-policy-weakening.ts | 2 + test/redact-declined.test.ts | 113 +++++++++++++++++++++++++ test/redact-default-union.test.ts | 5 +- test/redact-fixed-point.test.ts | 51 +++++++++++ 7 files changed, 241 insertions(+), 78 deletions(-) create mode 100644 test/redact-declined.test.ts create mode 100644 test/redact-fixed-point.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index fcbeb5803..7d3e169bc 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -6,7 +6,7 @@ - **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. A shape with a distinctive prefix (`ghp_`, `github_pat_`, `sk-ant-`, `xoxb-`, `lin_api_`, `AKIA`, `AIza`, `eyJ`, `wld_`, ...) is found wherever its prefix is, whatever is glued before it. The other shapes (a Discord bot token, a Bearer or Basic credential in any case, a Stripe key, a plain `sk-` key) are found after a name glued with `_`, an escape sequence (ESC as a byte, as `\u001b`, `\x1b`, `\033`, `\e` or `^[`, and the 8-bit CSI) or a JSON escape (`\n`). Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. - **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. This covers PEM keys (upper-case BEGIN and END lines with five dashes) and PGP's private key block. Not covered: a key whose BEGIN and END are both outside a window (only its middle lines in view), and other formats (an ssh.com SSH2 key, a lower-case PEM line, a PuTTY `.ppk` file). -- **The rules no longer run one after another.** Every rule reads the text as it was, and each run of what they find is replaced once, so one rule's match can't take in the text another rule starts from. Before, a dashboard key, a JWT, a Slack or Tailscale token or a bearer value written right before a URL with a password, a bearer token or a Linear key could hide it from the next rule (`wld_-redis://app:@…`). A JWT is found in any dotted part of a run (`sess_0123456789.`). Where one rule's find covers the others, it shows that rule's marker (a bearer token that is an `sk-` key now shows as `Bearer [REDACTED]`); where finds only overlap, the run shows `[REDACTED]`. A test holds the default mode to the previous release's rules on composed input. +- **The rules no longer run one after another.** Every rule reads the text as it was, and each run of what they find is replaced once, so one rule's match can't take in the text another rule starts from. Before, a dashboard key, a JWT, a Slack or Tailscale token or a bearer value written right before a URL with a password, a bearer token or a Linear key could hide it from the next rule (`wld_-redis://app:@…`). A JWT is found in any dotted part of a run (`sess_0123456789.`), an unsigned one (`alg: none`) too. An HTTP Basic value is found whatever is glued after it. Where one rule's find covers the others, it shows that rule's marker (a bearer token that is an `sk-` key now shows as `Bearer [REDACTED]`); where finds only overlap, the run shows `[REDACTED]`. A test holds the default mode to the previous release's rules on composed input. - **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. - **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. Each window ends at its last whitespace, so a token cut by the window's end is never shown. A text the redactor can't scan (a token prefix followed by a run of millions of characters) becomes `[REDACTED: too large to scan]`, and a chat attachment like that is refused as "text too large to scan" until it is attached anyway. The URL-password rule no longer slows down on a long dotted line. - A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 49e701fd6..97cb2f5e5 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -115,8 +115,9 @@ const sharedRedact = (() => { const exports = {}; "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.CUT_MARGIN = exports.UNSCANNABLE = exports.UNSCANNABLE_KIND = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; +exports.CUT_MARGIN = exports.UNSCANNABLE = exports.MAX_PASSES = exports.UNSCANNABLE_KIND = exports.TRACKER = exports.REDACTION_PATTERNS = exports.isDashboardKey = exports.DASHBOARD_KEY_PREFIX = void 0; exports.secretKinds = secretKinds; +exports.fixedPoint = fixedPoint; exports.redactString = redactString; exports.redactCut = redactCut; exports.readWindow = readWindow; @@ -161,17 +162,6 @@ function orphanEnd(m) { } return cut === -1 ? m : `[REDACTED PRIVATE KEY]${m.slice(cut)}`; } -function jwtIn(m, pre) { - const parts = m.slice(pre.length).split('.'); - let before = pre; - for (let i = 0; i + 2 < parts.length; i++) { - const at = parts[i].indexOf('eyJ'); - if (at !== -1 && parts[i].length - at >= 11 && parts[i + 1].length >= 8) - return `${before}${parts[i].slice(0, at)}[REDACTED JWT]`; - before += `${parts[i]}.`; - } - return m; -} const PATTERNS = [ [new RegExp(`${exports.DASHBOARD_KEY_PREFIX}${DASHBOARD_KEY_BODY}(?![A-Za-z0-9_-])`, 'g'), `${exports.DASHBOARD_KEY_PREFIX}[REDACTED]`, 'dashboard key'], [/gh[pousr]_[A-Za-z0-9]{30,}(?![A-Za-z0-9])/g, 'gh*_[REDACTED]', 'GitHub token'], @@ -180,7 +170,11 @@ const PATTERNS = [ [/\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/g, 'sk-[REDACTED]', 'API key (sk-)'], [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], - [/(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]+))\2\.(?=([A-Za-z0-9_-]+))\3\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, (m, pre) => jwtIn(m, pre), 'JWT'], + [ + /(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]*?)eyJ)\2eyJ(?=[A-Za-z0-9_-]{8})(?=([A-Za-z0-9_-]+))\3\.(?=[A-Za-z0-9_-]{8})(?=([A-Za-z0-9_-]+))\4\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]*)*/g, + '$1$2[REDACTED JWT]', + 'JWT', + ], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], [/xoxe[.-][A-Za-z0-9.-]{10,}(?![A-Za-z0-9])/g, 'xoxe-[REDACTED]', 'Slack token'], [/xox[abprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])/g, 'xox-[REDACTED]', 'Slack token'], @@ -191,7 +185,7 @@ const PATTERNS = [ [/-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----(?:[\s\S]*?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|[\s\S]*)/g, '[REDACTED PRIVATE KEY]', 'private key'], [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, orphanEnd, 'private key'], [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], - [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/gi, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], + [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})/gi, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], [ @@ -346,8 +340,10 @@ function spansOf(t, re, rep, rank, out) { g.lastIndex = Math.max(m.index + 1, m.index + whole.length - 64); const groups = m.slice(1).map((x) => x ?? ''); const r = typeof rep === 'string' ? expand(rep, whole, groups) : rep(whole, ...groups); - if (r === whole) + if (r === whole) { + g.lastIndex = m.index + 1; continue; + } let a = 0; while (a < whole.length && a < r.length && whole[a] === r[a]) a++; @@ -390,15 +386,19 @@ function replaceOnce(s, keepLines) { } return out + s.slice(at); } -function replaceUnion(s, keepLines) { - let out = replaceOnce(s, keepLines); - for (let i = 0; i < 2 && out !== s; i++) { - const again = replaceOnce(out, keepLines); +exports.MAX_PASSES = 8; +function fixedPoint(s, pass, keepLines = false) { + let out = s; + for (let i = 0; i < exports.MAX_PASSES; i++) { + const again = pass(out); if (again === out) - break; + return out; out = again; } - return out; + return exports.UNSCANNABLE + (keepLines ? '\n'.repeat(breaks(s)) : ''); +} +function replaceUnion(s, keepLines) { + return fixedPoint(s, (t) => replaceOnce(t, keepLines), keepLines); } function maskOnce(s, mask, keepLines) { const spans = allSpans(s, BROAD_RULES); @@ -488,14 +488,7 @@ function outside(spans, regions) { return out; } function maskUnion(s, mask, keepLines) { - let out = maskOnce(s, mask, keepLines); - for (let i = 0; i < 2; i++) { - const again = maskOnce(out, mask, keepLines); - if (again === out) - break; - out = again; - } - return out; + return fixedPoint(s, (t) => maskOnce(t, mask, keepLines), keepLines); } exports.UNSCANNABLE = '[REDACTED: too large to scan]'; function redactString(s, o = {}) { diff --git a/src/events/redact.ts b/src/events/redact.ts index 2d362fd56..841d2f032 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -79,24 +79,6 @@ function orphanEnd(m: string): string { return cut === -1 ? m : `[REDACTED PRIVATE KEY]${m.slice(cut)}`; } -/** - * `m` (a run of three or more dotted parts after `pre`) with a JWT in it redacted, from its `eyJ` to the run's end: the first part - * holding `eyJ` and 8 more characters, followed by a part of 8 or more (the payload) and one more dot. Any part may be - * the header (#889 round 6: `name.eyJ...`, where the run's first part is a name, was found by main's rule from the - * dot); within a part, the header may be glued after anything (round 5). As it was when no part is a header. One walk - * over the parts: linear. - */ -function jwtIn(m: string, pre: string): string { - const parts = m.slice(pre.length).split('.'); - let before = pre; - for (let i = 0; i + 2 < parts.length; i++) { - const at = parts[i]!.indexOf('eyJ'); - if (at !== -1 && parts[i]!.length - at >= 11 && parts[i + 1]!.length >= 8) return `${before}${parts[i]!.slice(0, at)}[REDACTED JWT]`; - before += `${parts[i]}.`; - } - return m; -} - // Each pattern with what kind of secret it finds (secretKinds names the kind, never the value). const PATTERNS: [RegExp, Replacement, string][] = [ // A dashboard key, wherever its prefix is (after a `-` or a letter too: `key-wld_...`): a match runs to the end of its @@ -116,15 +98,22 @@ const PATTERNS: [RegExp, Replacement, string][] = [ [/(^|[^A-Za-z0-9_])([sr]k)_(?:live|test)_([A-Za-z0-9]{16,})(?![A-Za-z0-9_])/g, (m, pre, k, v) => (/\d/.test(v!) ? `${pre}${k}_[REDACTED]` : m), 'Stripe key'], // Google API keys: AIza and 35 more characters. [/AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, 'AIza[REDACTED]', 'Google API key'], - // JSON Web Tokens: a base64url JSON header (so `eyJ`), a payload and a signature (empty when unsigned). - // Any further dotted parts go with it (#889 round 3): a JWE has five, and a token may be followed by more. - // Read from the start of the header's run of base64url characters, with no bound on its length (an `x5c` chain makes - // a header several KB, #889 round 5); a function finds `eyJ` in that run, so a token glued after a letter, a digit or - // `-` is found from its `eyJ` on. Each start is a run's start: linear. - // Each of the first two parts is read whole, with no way back into it (a lookahead takes the run, a backreference - // steps over it): a run of millions of characters with no dot after it is passed in one step, not given back a - // character at a time until V8's backtracking stack overflows (#889 round 6). jwtIn checks the parts' lengths. - [/(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]+))\2\.(?=([A-Za-z0-9_-]+))\3\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*/g, (m, pre) => jwtIn(m, pre!), 'JWT'], + // JSON Web Tokens: a base64url JSON header (so `eyJ` and 8 or more), a payload of 8 or more and a signature (empty + // when unsigned, or cut at a `+` or `/`). Any further dotted parts go with it, empty ones too (#889 rounds 3 and 7): a + // JWE has five, and a token may be followed by more. + // - The header is found in any part, glued after anything in it (`x`, `%3D`, `name.`): the match + // starts at a part's start (after a character no part holds: `.`, a space, `%`), and a lookahead looks in that part + // for `eyJ`. It can't leave its part, so each part is read once from its start: linear. + // - Each of the header and the payload is read whole, with no way back into it (a lookahead takes the run, a + // backreference steps over it), with no bound on its length (an `x5c` chain makes a header several KB, round 5): + // a run of millions of characters is passed in one step, not given back a character at a time until V8's + // backtracking stack overflows (round 6). + // - A plain replacement: no match is declined, so nothing the search steps over can hide a token (round 7). + [ + /(^|[^A-Za-z0-9_-])(?=([A-Za-z0-9_-]*?)eyJ)\2eyJ(?=[A-Za-z0-9_-]{8})(?=([A-Za-z0-9_-]+))\3\.(?=[A-Za-z0-9_-]{8})(?=([A-Za-z0-9_-]+))\4\.[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]*)*/g, + '$1$2[REDACTED JWT]', + 'JWT', + ], [/(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/g, 'AKIA[REDACTED]', 'AWS access key'], // Slack's token-rotation and refresh tokens (xoxe-..., xoxe.xoxb-...), before the // other Slack tokens (one is inside it); then its app-level token (xapp-, Socket Mode). @@ -147,8 +136,10 @@ const PATTERNS: [RegExp, Replacement, string][] = [ [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], // HTTP Basic credentials (base64 of user:password; git's extraheader, src/dashboard-auth.ts's client id and secret), // failing closed: any credential-looking value after the scheme. Prose ("Basic Authentication") has no digit, `+`, - // `/` or `=`, and isn't a mixed-case run of a base64 length. - [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})(?![A-Za-z0-9+/=])/gi, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], + // `/` or `=`, and isn't a mixed-case run of a base64 length. The value ends where its run and padding end, whatever is + // glued after it (#889 round 7: a check that nothing followed left `Basic ==` to a later pass, after + // another rule's marker made the boundary; broad mode, which masks that rule's text, showed it). + [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})/gi, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], // Credentials inside connection strings: keep scheme, user and host. A scheme of up to 32 characters right before // `://`, from any start (#889 round 4: from `\b` with any length it was quadratic on a long dotted run): each start // reads at most 33 characters, and each `://` is reached from at most 32 starts, so linear. @@ -435,7 +426,13 @@ function spansOf(t: string, re: RegExp, rep: Replacement, rank: number, out: Spa g.lastIndex = Math.max(m.index + 1, m.index + whole.length - 64); const groups = m.slice(1).map((x) => x ?? ''); const r = typeof rep === 'string' ? expand(rep, whole, groups) : rep(whole, ...groups); - if (r === whole) continue; + if (r === whole) { + // A match the rule's function declined (no secret) hides nothing it stepped over (#889 round 7): the next search + // starts at its next character, so a start inside it is still tried. Linear only for a rule whose starts are + // bounded (a prefix group or a quote): test/redact-declined.test.ts holds every function-backed rule to both. + g.lastIndex = m.index + 1; + continue; + } let a = 0; while (a < whole.length && a < r.length && whole[a] === r[a]) a++; let b = 0; @@ -486,18 +483,28 @@ function replaceOnce(s: string, keepLines: boolean): string { return out + s.slice(at); } +/** How many passes a mode may take to reach a text no pass changes (fixedPoint). */ +export const MAX_PASSES = 8; + /** - * The default mode: replaced once, then again on what that gave, until nothing changes (at most 3 times). A marker can - * make a boundary a later rule needs, as main's rules, applied in turn, had. + * `pass` applied to `s`, then to what it gave, until it changes nothing, failing closed (#889 round 7): a text still + * changing after MAX_PASSES passes becomes UNSCANNABLE (with `s`'s line breaks when `keepLines`), never the last pass's + * output, which a further pass would have redacted more. Each pass reads the whole text: at most MAX_PASSES times + * linear. No input measured so far needs a third pass. */ -function replaceUnion(s: string, keepLines: boolean): string { - let out = replaceOnce(s, keepLines); - for (let i = 0; i < 2 && out !== s; i++) { - const again = replaceOnce(out, keepLines); - if (again === out) break; +export function fixedPoint(s: string, pass: (t: string) => string, keepLines = false): string { + let out = s; + for (let i = 0; i < MAX_PASSES; i++) { + const again = pass(out); + if (again === out) return out; out = again; } - return out; + return UNSCANNABLE + (keepLines ? '\n'.repeat(breaks(s)) : ''); +} + +/** The default mode: replaced again on what each pass gave until nothing changes. A marker can make a boundary a later rule needs, as main's rules, applied in turn, had. */ +function replaceUnion(s: string, keepLines: boolean): string { + return fixedPoint(s, (t) => replaceOnce(t, keepLines), keepLines); } /** @@ -602,17 +609,11 @@ function outside(spans: [number, number][], regions: [number, number][]): [numbe } /** - * The broad mode: masked once, then again on what that gave, until nothing changes (at most 3 times). A mask can make - * a boundary a later rule needs (`[REDACTED]_db_password=...`), as each module's own rules, applied in turn, did. + * The broad mode: masked again on what each pass gave until nothing changes (fixedPoint). A mask can make a boundary a + * later rule needs (`[REDACTED]_db_password=...`), as each module's own rules, applied in turn, did. */ function maskUnion(s: string, mask: string, keepLines: boolean): string { - let out = maskOnce(s, mask, keepLines); - for (let i = 0; i < 2; i++) { - const again = maskOnce(out, mask, keepLines); - if (again === out) break; - out = again; - } - return out; + return fixedPoint(s, (t) => maskOnce(t, mask, keepLines), keepLines); } /** What a text becomes when it can't be scanned (#889 round 5): shown as nothing of it. */ diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 9072e7421..64ab9e16c 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -99,6 +99,8 @@ export const CHECK_TABLES: readonly CheckTable[] = [ // Main's default rules, the baseline the default mode must hide at least as much as (#889 round 6). { path: 'test/redact-default-union.test.ts', table: 'MAIN_PATTERNS', kind: 'never-shrink' }, { path: 'test/redact-default-union.test.ts', table: 'CREDENTIAL_WORDS', kind: 'never-shrink' }, + { path: 'test/redact-default-union.test.ts', table: 'KEY_QUALIFIERS', kind: 'never-shrink' }, + { path: 'test/redact-default-union.test.ts', table: 'CREDENTIAL_ENDINGS', kind: 'never-shrink' }, // The guards' own tables (their files are guard files too; the table names what was taken out). { path: 'src/review.ts', table: 'SELF_MODIFICATION_PARTS', kind: 'never-shrink' }, { path: 'src/review.ts', table: 'LOCKED_CATEGORIES', kind: 'never-shrink' }, diff --git a/test/redact-declined.test.ts b/test/redact-declined.test.ts new file mode 100644 index 000000000..336072d5d --- /dev/null +++ b/test/redact-declined.test.ts @@ -0,0 +1,113 @@ +// #889 round 7 (the round-6 second review, A1): a match a rule's function declines (no secret) must not hide a token +// that starts inside it. An unsigned JWT (`alg: none`, an empty third part) or one whose signature starts with `+` or +// `/`, written right after a dotted name (`a..`), was shown: the JWT rule's match stopped before the empty part, +// its function declined the match, and the next search started 64 characters before the match's end, past the +// JWT's own start. Now the JWT rule declines nothing (a plain replacement), and a declined match of any rule restarts +// at its next character. Every function-backed rule is held here to a reference that tries every start. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { REDACTION_PATTERNS, redactCut, redactString, secretKinds } from '../src/events/redact.js'; +import { run } from './token-standins.js'; + +const KEY_BEGIN = ['-----BEGIN', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const KEY_END = ['-----END', 'RSA', 'PRIVATE', 'KEY-----'].join(' '); +const broad = { broad: true } as Parameters[1]; + +test('an unsigned or `+`-led JWT after a dotted name is found: default, broad, redactCut and secretKinds', () => { + const missed: string[] = []; + for (const pre of ['a.', 'x.', 'a.b.', 'x.y.z.', 'api.example-service.', ['sess', '0123456789.'].join('_'), 'id=a.', '"a.']) { + for (const [h, p] of [[8, 8], [8, 40], [8, 70], [30, 40], [60, 20], [100, 100]] as const) { + for (const tail of ['.', '. ', '.\n', '..x', '."', '.!', '.+Sg4Qe9Ja2Wz7', './Sg4Qe9Ja2Wz7']) { + const payload = `P${run(p - 1, 'Ab3Kq7Zx')}`; + const s = `${pre}${'eyJ'}${run(h, 'Hd3Rz7Kq')}.${payload}${tail}`; + const where = `${JSON.stringify(pre)} h=${h} p=${p} ${JSON.stringify(tail)}`; + if (redactString(s).includes(payload)) missed.push(`default ${where}`); + if (redactString(s, broad).includes(payload)) missed.push(`broad ${where}`); + if (redactCut(s, 100_000).includes(payload)) missed.push(`redactCut ${where}`); + if (!secretKinds(s).includes('JWT')) missed.push(`secretKinds ${where}`); + } + } + } + assert.deepEqual(missed.slice(0, 20), []); +}); + +/** + * Where `rule` finds a secret in `s` from any start: the text each accepted match's replacement takes out, with where + * it is, as `taken@position` (each once). + */ +function everyStart(s: string, re: RegExp, rep: (m: string, ...g: string[]) => string): Set { + const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`); + const out = new Set(); + for (let p = 0; p < s.length; p++) { + g.lastIndex = p; + const m = g.exec(s); + if (!m || m.index !== p) continue; + const r = rep(m[0], ...m.slice(1).map((x) => x ?? '')); + if (r === m[0]) continue; + let a = 0; + while (a < m[0].length && a < r.length && m[0][a] === r[a]) a++; + let b = 0; + while (b < m[0].length - a && b < r.length - a && m[0][m[0].length - 1 - b] === r[r.length - 1 - b]) b++; + const taken = m[0].slice(a, m[0].length - b); + if (taken.length >= 8) out.add(`${taken}@${p + a}`); + } + return out; +} + +const occurrences = (t: string, x: string) => t.split(x).length - 1; + +/** Pieces each function-backed rule declines or accepts, by the rule's kind. */ +const PIECES: Record = { + 'Stripe key': [`${'sk'}_live_`, `${'rk'}_test_`, 'abcdefghijklmnopqrst', run(18, 'Q7rZ4kX2m9TbW3pL'), ' ', '-', '.', '_', 'x'], + 'secret under a JSON key': ['"', '\\"', 'key', 'cacheKey', 'token', 'api_key', ':', ': ', 'issue-644-aaaaaaaaaaaa', run(20, 'Q7rZ4kX2m9TbW3pL'), ','], + // A Basic value (base64 of `u:` and a password), put together at run time. + 'Basic credential': ['Basic ', 'basic ', 'Authentication', Buffer.from(['u', 'Ch8641A7E2'].join(':')).toString('base64'), '==', '=', ' ', 'x', '+'], + 'private key': [KEY_BEGIN, KEY_END, '\n', 'body', ' '], +}; + +test('every function-backed rule is covered here, and nothing it accepts from any start is shown', () => { + const rules = REDACTION_PATTERNS.filter((p) => p.group === 'secret' && typeof p.rep === 'function'); + // A new function-backed rule must be added to PIECES: its declined matches are where a token can hide. + assert.deepEqual([...new Set(rules.map((p) => p.kind))].sort(), Object.keys(PIECES).sort()); + const missed: string[] = []; + let seed = 7; + const rnd = (k: number) => { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return (seed >>> 8) % k; + }; + for (const rule of rules) { + const pieces = [...PIECES[rule.kind]!, ...PIECES['Basic credential']!.slice(0, 2), '"', ' ']; + for (let i = 0; i < 4000; i++) { + let s = ''; + for (let j = 2 + rnd(7); j > 0; j--) s += pieces[rnd(pieces.length)]; + const out = redactString(s); + // Each place a match is accepted from some start is one occurrence the output must have lost (redaction never + // makes a new one). + const hidden = new Map(); + for (const at of everyStart(s, rule.re, rule.rep as (m: string, ...g: string[]) => string)) { + const taken = at.slice(0, at.lastIndexOf('@')); + hidden.set(taken, (hidden.get(taken) ?? 0) + 1); + } + for (const [taken, n] of hidden) { + if (occurrences(out, taken) > occurrences(s, taken) - n && missed.length < 20) missed.push(`${rule.kind}: ${JSON.stringify(s)} shows ${JSON.stringify(taken)}`); + } + } + } + assert.deepEqual(missed, []); +}); + +test('declined matches restart cheaply: 1 MB of each rule\'s declined text in under 2 s (default and broad)', () => { + const declined = ['"key":"issue-644-aaaaaaaaaaaa",', '"a":"', `${'sk'}_live_abcdefghijklmnopqrst `, 'Basic Authentication ', `${KEY_END}\n`, 'a.b.c.', `${'eyJ'}aaaaaaaa.x.`]; + const slow: string[] = []; + for (const unit of declined) { + const s = unit.repeat(Math.ceil(1_000_000 / unit.length)); + for (const [mode, o] of [['default', {}], ['broad', broad]] as const) { + const t0 = performance.now(); + redactString(s, o); + const ms = performance.now() - t0; + if (ms > 2000) slow.push(`${mode} ${JSON.stringify(unit)}: ${Math.round(ms)} ms`); + } + } + assert.deepEqual(slow, []); +}); diff --git a/test/redact-default-union.test.ts b/test/redact-default-union.test.ts index e7cb533a3..e95308a14 100644 --- a/test/redact-default-union.test.ts +++ b/test/redact-default-union.test.ts @@ -75,6 +75,9 @@ const SHAPES: [string, string, string][] = (() => { ['Slack app token', `${'xapp'}-1-${s(30, 'Xa9Pp2Tk5Mr7')}`, s(30, 'Xa9Pp2Tk5Mr7')], ['Linear key', `${'lin'}_api_${s(40, 'Ln3Ar8Pk6Qc2')}`, s(40, 'Ln3Ar8Pk6Qc2')], ['JWT', `${'eyJ'}${s(20, 'hDr1Bq5Zt8')}.${'eyJ'}${s(24, 'pLd4Nx7Jv2')}.${jwtSig}`, jwtSig], + // Unsigned (`alg: none`: an empty third part), and one whose signature starts with `+` (#889 round 7). + ['unsigned JWT', `${'eyJ'}${s(20, 'hDr1Bq5Zt8')}.${'eyJ'}${s(60, 'uNs7Gd2Pq9Lw')}.`, s(60, 'uNs7Gd2Pq9Lw')], + ['JWT with a +-led signature', `${'eyJ'}${s(20, 'hDr1Bq5Zt8')}.${'eyJ'}${s(60, 'pLs4Gn8Wq3Zk')}.+${s(20, 'Sx5Rt1Mv7Yb2')}`, s(60, 'pLs4Gn8Wq3Zk')], ['AWS key', `${'AKIA'}${'Q7RZ4KX2M9TBH3WD'}`, 'Q7RZ4KX2M9TBH3WD'], ['Google key', `${'AIza'}${s(35, 'Go4Gk1Ey9Ap3')}`, s(35, 'Go4Gk1Ey9Ap3')], ['Tailscale key', `${'tskey'}-auth-${s(30, 'Ts2Ak7Qz4Wf8')}`, s(30, 'Ts2Ak7Qz4Wf8')], @@ -87,7 +90,7 @@ const SHAPES: [string, string, string][] = (() => { })(); /** Separators a log line, a URL, JSON or an ANSI-coloured terminal puts between two values. */ -const SEPARATORS = ['', ' ', '-', '.', '+', '_', ':', '=', ',', '/', '"', '\n', '\u001b[1m', '\\n', 'Authorization: ', 'BEARER ']; +const SEPARATORS = ['', ' ', '-', '.', '+', '_', ':', '=', ',', '/', '"', '\n', '\u001b[1m', '\\n', 'Authorization: ', 'BEARER ', 'x.', 'a.b.']; /** The secrets main hid in `s` that `out` shows. */ function narrowed(s: string, out: string, secrets: string[]): string[] { diff --git a/test/redact-fixed-point.test.ts b/test/redact-fixed-point.test.ts new file mode 100644 index 000000000..bce20137e --- /dev/null +++ b/test/redact-fixed-point.test.ts @@ -0,0 +1,51 @@ +// #889 round 7 (the round-6 delta, Q3): the passes fail closed. Each mode passes over its own output until nothing +// changes; a text still changing after the bound becomes the placeholder, never the last pass's output (which one more +// pass would have redacted further). Before, each mode stopped after 3 passes whatever a 4th would have found. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import * as redactModule from '../src/events/redact.js'; +import { redactString, UNSCANNABLE } from '../src/events/redact.js'; + +type FixedPoint = (s: string, pass: (t: string) => string, keepLines?: boolean) => string; +const fixedPoint = (redactModule as unknown as { fixedPoint?: FixedPoint }).fixedPoint; +const MAX_PASSES = (redactModule as unknown as { MAX_PASSES?: number }).MAX_PASSES; + +test('a pass that keeps changing the text past the bound gives the placeholder, line breaks kept when asked', () => { + assert.equal(typeof fixedPoint, 'function'); + assert.ok(typeof MAX_PASSES === 'number' && MAX_PASSES > 3); + const shrink = (t: string) => t.slice(1); + assert.equal(fixedPoint!('a'.repeat(100), shrink), UNSCANNABLE); + assert.equal(fixedPoint!(`a\nb\n${'c'.repeat(100)}`, shrink, true), `${UNSCANNABLE}\n\n`); +}); + +test('a text that needs more than 3 passes, within the bound, gets them all', () => { + let passes = 0; + const strip = (t: string) => { + passes++; + return t.replace(/^x/, ''); + }; + // Four x's: four passes change it, a fifth confirms nothing changes. + assert.equal(fixedPoint!('xxxxdone', strip), 'done'); + assert.equal(passes, 5); +}); + +test('both modes reach a fixed point: redacting the output again changes nothing (generated, glued pieces)', () => { + const T = 'pL0oK9iJ8uH7yG6tF5rD4eS3Mn3Bv4Cx5Zl6Kj7H'; + const PIECES = [`${'ghp'}_`, `${'github'}_pat_`, `${'sk'}-ant-`, `${'sk'}-`, `${'xoxb'}-`, `${'eyJ'}`, 'AKIA', `${'wld'}_`, `${'tskey'}-`, 'Bearer ', 'basic ', 'https://', 'u:', '@', '.', '-', '_', ' ', '"', ':', '=', '\n', '\u001b[1m', '\\n', '[REDACTED]', ']', '[', 'token=', '"api_key":"', T, T.slice(0, 16), T.slice(0, 8)]; + let seed = 777; + const rnd = (k: number) => { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return (seed >>> 8) % k; + }; + const moved: string[] = []; + for (let i = 0; i < 20_000; i++) { + let s = ''; + for (let j = 2 + rnd(9); j > 0; j--) s += PIECES[rnd(PIECES.length)]; + for (const o of [{}, { broad: true }]) { + const once = redactString(s, o); + if (redactString(once, o) !== once && moved.length < 10) moved.push(JSON.stringify(s)); + } + } + assert.deepEqual(moved, []); +}); From 71d5d4a43a3c127cc3a53e4caaa33fa4d5332a97 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 12:27:31 -0500 Subject: [PATCH 9/9] redaction: any-length URL scheme, login link first, short Basic values, helper tables marked (#889) Round 8, on the round-7 second review (#921): - The URL-password rule and the broad user-info rule read the last 1 to 32 characters of the scheme, with no letter required: a scheme longer than 32 characters whose last 32 are digits, `-` or `.` hid the password from both, where main's rule (a scheme of any length) found it. Still linear (each `://` is reached from at most 32 starts). - src/tailscale-setup.ts replaces the login link before redacting, as main did: a token glued before the link (`tskey-...https://login...`) took in its `https` and the link was shown. - basicShaped also takes a value that decodes to printable text with a `:` (a short user:password with no digit, `+`, `/` or `=`); prose ("Basic Authentication") doesn't decode to that. - GLUED_PREFIXES, KEY_QUALIFIERS, CREDENTIAL_ENDINGS and the glue sequences (now GLUE_PARTS, an array) carry never-shrink marks, in the module and, through the generator, in the updater's copy. SHAPES has a plain `sk-` key. - test/redact-default-union.test.ts adds a long-scheme URL password. Test (fails without): test/redact-round8.test.ts, 4 of 4 failing at fa1318fd. Signed-off-by: Jayden Vasquez --- changelog.d/921-one-shared-redactor.md | 4 +- scripts/machine/worklane-update.cjs | 25 ++++++-- src/events/redact.ts | 60 ++++++++++++------- src/tailscale-setup.ts | 6 +- test/redact-default-union.test.ts | 2 + test/redact-round8.test.ts | 82 ++++++++++++++++++++++++++ test/token-standins.ts | 2 + 7 files changed, 151 insertions(+), 30 deletions(-) create mode 100644 test/redact-round8.test.ts diff --git a/changelog.d/921-one-shared-redactor.md b/changelog.d/921-one-shared-redactor.md index 7d3e169bc..f51cc3027 100644 --- a/changelog.d/921-one-shared-redactor.md +++ b/changelog.d/921-one-shared-redactor.md @@ -6,7 +6,7 @@ - **One shared list of token shapes (#889).** The engine's redactor (`src/events/redact.ts`) now knows a Discord bot token, a Tailscale key, a dashboard key and an HTTP Basic credential. A shape with a distinctive prefix (`ghp_`, `github_pat_`, `sk-ant-`, `xoxb-`, `lin_api_`, `AKIA`, `AIza`, `eyJ`, `wld_`, ...) is found wherever its prefix is, whatever is glued before it. The other shapes (a Discord bot token, a Bearer or Basic credential in any case, a Stripe key, a plain `sk-` key) are found after a name glued with `_`, an escape sequence (ESC as a byte, as `\u001b`, `\x1b`, `\033`, `\e` or `^[`, and the 8-bit CSI) or a JSON escape (`\n`). Before, Discord and Tailscale were each redacted only by one module of their own, so a Discord bot token passed through to the event log, issue comments, chat notices and the dashboard's service log. - **A private key fails closed.** A key's BEGIN line with no END after it is redacted to the end of the text, and an END line with no BEGIN before it from the start, so a key cut short, still being written, or begun above a log window is not shown. This covers PEM keys (upper-case BEGIN and END lines with five dashes) and PGP's private key block. Not covered: a key whose BEGIN and END are both outside a window (only its middle lines in view), and other formats (an ssh.com SSH2 key, a lower-case PEM line, a PuTTY `.ppk` file). -- **The rules no longer run one after another.** Every rule reads the text as it was, and each run of what they find is replaced once, so one rule's match can't take in the text another rule starts from. Before, a dashboard key, a JWT, a Slack or Tailscale token or a bearer value written right before a URL with a password, a bearer token or a Linear key could hide it from the next rule (`wld_-redis://app:@…`). A JWT is found in any dotted part of a run (`sess_0123456789.`), an unsigned one (`alg: none`) too. An HTTP Basic value is found whatever is glued after it. Where one rule's find covers the others, it shows that rule's marker (a bearer token that is an `sk-` key now shows as `Bearer [REDACTED]`); where finds only overlap, the run shows `[REDACTED]`. A test holds the default mode to the previous release's rules on composed input. +- **The rules no longer run one after another.** Every rule reads the text as it was, and each run of what they find is replaced once, so one rule's match can't take in the text another rule starts from. Before, a dashboard key, a JWT, a Slack or Tailscale token or a bearer value written right before a URL with a password, a bearer token or a Linear key could hide it from the next rule (`wld_-redis://app:@…`). A JWT is found in any dotted part of a run (`sess_0123456789.`), an unsigned one (`alg: none`) too. An HTTP Basic value is found whatever is glued after it, and a short one too (base64 of a short `user:password`). A password in a URL is found whatever the scheme's length. Where one rule's find covers the others, it shows that rule's marker (a bearer token that is an `sk-` key now shows as `Bearer [REDACTED]`); where finds only overlap, the run shows `[REDACTED]`. A test holds the default mode to the previous release's rules on composed input. - **No module keeps its own token rules.** The GitHub sign-in relay, the bench and the root updater now redact through the shared redactor. Its broad mode runs every rule on the text as it was and masks what any of them finds. That covers everything those modules masked before: a token cut short, a clone URL's user info, a value after a secret's name, and a short bearer token. A test compares each with its rules on the previous release, on generated input. The updater is one standalone file, so it carries a generated copy of the redactor, and a test holds that copy to the module. - **The dashboard's service log** redacts its journal entries and log-file lines as one text, with 100 more above the ones shown, and redacts a long line before cutting it to 2,000 characters. A PR section, a change's diff and a chat attachment are redacted before they are cut, too. Each of these reads only what it keeps plus a margin, so a huge input costs no more than its window. Each window ends at its last whitespace, so a token cut by the window's end is never shown. A text the redactor can't scan (a token prefix followed by a run of millions of characters) becomes `[REDACTED: too large to scan]`, and a chat attachment like that is refused as "text too large to scan" until it is attached anyway. The URL-password rule no longer slows down on a long dotted line. -- A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`). A test fails on any module that redacts a token shape with its own pattern. +- A Tailscale key in a Tailscale step line now shows as `tskey-[REDACTED]` (was `[a key]`); the login link is still taken out first, so a token glued before it can't hide it. A test fails on any module that redacts a token shape with its own pattern. diff --git a/scripts/machine/worklane-update.cjs b/scripts/machine/worklane-update.cjs index 653f7ea57..597320052 100644 --- a/scripts/machine/worklane-update.cjs +++ b/scripts/machine/worklane-update.cjs @@ -124,11 +124,23 @@ exports.readWindow = readWindow; exports.redactRecords = redactRecords; exports.redact = redact; const credentialShaped = (v) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && /\d/.test(v)) || /^[0-9a-fA-F]{32,}$/.test(v); -const basicShaped = (v) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)); +const basicShaped = (v) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)) || basicPair(v); +function basicPair(v) { + let text; + try { + text = atob(v); + } + catch { + return false; + } + return /^[\x20-\x7e]*:[\x20-\x7e]*$/.test(text); +} // check-table: never-shrink const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'jwt', 'bearer', 'credential', 'credentials', 'authorization']); +// check-table: never-shrink const CREDENTIAL_ENDINGS = ['token', 'secret', 'password', 'passwd', 'accesskey']; const credentialWord = (w) => CREDENTIAL_WORDS.has(w) || CREDENTIAL_ENDINGS.some((e) => w.endsWith(e)); +// check-table: never-shrink const KEY_QUALIFIERS = new Set(['api', 'access', 'secret', 'private', 'signing', 'encryption', 'master', 'client', 'app', 'auth', 'account', 'service', 'license', 'deploy', 'admin', 'webhook', 'session', 'ssh', 'hmac', 'consumer']); function credentialKey(k) { const words = k @@ -188,7 +200,7 @@ const PATTERNS = [ [/^[\s\S]*-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/, orphanEnd, 'private key'], [/\b(bearer)\s+[A-Za-z0-9._~+/-]{20,}=*/gi, '$1 [REDACTED]', 'bearer token'], [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})/gi, (m, s, sp, v) => (basicShaped(v) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], - [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], + [/([a-z0-9+.-]{1,32}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], [ /(\\?")([A-Za-z0-9_-]+)\1(\s*:\s*)\1([A-Za-z0-9_\-.+/=~]{16,})\1/g, @@ -205,7 +217,7 @@ const BROAD = [ [/xox[abprs]-[A-Za-z0-9_-]+/g, (k) => k, 'Slack token, cut short'], [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => k, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], - [/([a-z][a-z0-9+.-]{0,31}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], + [/([a-z0-9+.-]{1,32}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], ]; const CODE_HEADS = new Set(['this', 'self', 'super', 'opts', 'options', 'o', 'ctx', 'context', 'config', 'cfg', 'conf', 'settings', 'props', 'state', 'env', 'process', 'module', 'exports', 'window', 'document', 'globalThis', 'args', 'params', 'req', 'res', 'request', 'response', 'obj', 'result', 'item', 'e', 'err', 'x']); @@ -242,12 +254,15 @@ exports.REDACTION_PATTERNS = [ ...BROAD.map(([re, rep, kind]) => ({ kind, re, rep: rep('[REDACTED]'), group: 'broad' })), ...IDENTIFIERS.map(([re, rep, kind]) => ({ kind, re, rep, group: 'identifier' })), ]; +// check-table: never-shrink const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', exports.DASHBOARD_KEY_PREFIX]; -const GLUE = new RegExp([ +// check-table: never-shrink +const GLUE_PARTS = [ String.raw `(?:\u001b|\\u001[bB]|\\x1[bB]|\\033|\\e|\^\[)(?:\[[0-?]*[ -/]*[@-~]|[ -/]+[0-~]|[0-~])`, String.raw `\u009b[0-?]*[ -/]*[@-~]`, String.raw `\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])`, -].join('|'), 'g'); +]; +const GLUE = new RegExp(GLUE_PARTS.join('|'), 'g'); function gluePoints(s) { const cuts = [...s.matchAll(GLUE)].map((m) => m.index + m[0].length); for (let at = s.indexOf('_'); at !== -1; at = s.indexOf('_', at + 1)) diff --git a/src/events/redact.ts b/src/events/redact.ts index fc1c1a954..484cddec6 100644 --- a/src/events/redact.ts +++ b/src/events/redact.ts @@ -18,7 +18,21 @@ const credentialShaped = (v: string) => (/[a-z]/.test(v) && /[A-Z]/.test(v) && / * A value after `Basic` that is a credential, not a word: base64 with a digit, `+`, `/` or `=` in it, or a mixed-case * run of 16 or more characters whose length base64 can have (no padding: not 1 more than a multiple of 4). */ -const basicShaped = (v: string) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)); +const basicShaped = (v: string) => /[0-9+/=]/.test(v) || (v.length >= 16 && v.length % 4 !== 1 && /[a-z]/.test(v) && /[A-Z]/.test(v)) || basicPair(v); +/** + * Whether `v` is base64 of a `user:password` pair: it decodes to printable ASCII with a `:` in it (#889 round 8: a short + * pair, `dXNlcjpwYXNz`, has no digit, `+`, `/` or `=` and is under 16 characters). A word after "Basic" in prose + * ("Authentication") decodes to bytes that aren't, or doesn't decode at all. + */ +function basicPair(v: string): boolean { + let text: string; + try { + text = atob(v); + } catch { + return false; + } + return /^[\x20-\x7e]*:[\x20-\x7e]*$/.test(text); +} /** A word that makes any key holding it a credential's: `db_password`, `slack_bot_token`, `aws_secret_access_key`. */ // check-table: never-shrink @@ -28,9 +42,11 @@ const CREDENTIAL_WORDS = new Set(['token', 'secret', 'secrets', 'password', 'pas * (`csrftoken`, `clientsecret`, `adminpassword`, `secretaccesskey`). Only the end counts, so `tokenizer` and * `secretary` are not; `pass` and `pwd` count only whole (`compass`, `bypass`). */ +// check-table: never-shrink const CREDENTIAL_ENDINGS = ['token', 'secret', 'password', 'passwd', 'accesskey']; const credentialWord = (w: string) => CREDENTIAL_WORDS.has(w) || CREDENTIAL_ENDINGS.some((e) => w.endsWith(e)); /** The word before `key` that makes it a credential (`x-api-key`, `apiKey`, `signing_key`); `cacheKey`, `sort_key` are not. */ +// check-table: never-shrink const KEY_QUALIFIERS = new Set(['api', 'access', 'secret', 'private', 'signing', 'encryption', 'master', 'client', 'app', 'auth', 'account', 'service', 'license', 'deploy', 'admin', 'webhook', 'session', 'ssh', 'hmac', 'consumer']); /** @@ -142,10 +158,12 @@ const PATTERNS: [RegExp, Replacement, string][] = [ // glued after it (#889 round 7: a check that nothing followed left `Basic ==` to a later pass, after // another rule's marker made the boundary; broad mode, which masks that rule's text, showed it). [/\b(basic)(\s+)([A-Za-z0-9+/]{8,}={0,2})/gi, (m, s, sp, v) => (basicShaped(v!) ? `${s}${sp}[REDACTED]` : m), 'Basic credential'], - // Credentials inside connection strings: keep scheme, user and host. A scheme of up to 32 characters right before - // `://`, from any start (#889 round 4: from `\b` with any length it was quadratic on a long dotted run): each start - // reads at most 33 characters, and each `://` is reached from at most 32 starts, so linear. - [/([a-z][a-z0-9+.-]{0,31}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], + // Credentials inside connection strings: keep scheme, user and host. The last 1 to 32 characters of the scheme right + // before `://`, from any start (#889 round 4: from `\b` with any length it was quadratic on a long dotted run): each + // start reads at most 33 characters, and each `://` is reached from at most 32 starts, so linear. No letter is + // required among them (round 8: a scheme whose last 32 characters are digits or `-` was main's too), so every match + // main's rule made, with at least one scheme character before `://`, this one makes. + [/([a-z0-9+.-]{1,32}:\/\/[^:/\s@]+):[^@\s/]+@/gi, '$1:[REDACTED]@', 'password in a URL'], // 0x-prefixed 32-byte hex (EVM private keys and the like). [/\b0x[0-9a-fA-F]{64}(?![A-Za-z0-9])/g, '0x[REDACTED-32-BYTES]', '32-byte hex key'], // A JSON string value under a credential key (credentialKey: "aws_secret_access_key", "db_password", "x-api-key", @@ -186,10 +204,10 @@ const BROAD: [RegExp, (mask: string) => string, string][] = [ [/xox[abprs]-[A-Za-z0-9_-]+/g, (k) => k, 'Slack token, cut short'], [/eyJ[A-Za-z0-9_-]{8,512}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*/g, (k) => k, 'JWT after a hyphen'], [/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, (k) => `$1 ${k}`, 'short bearer token'], - // A scheme of up to 32 characters right before `://`, from any start (after `-`, `.`, `+` or a letter too: the - // updater's own rule started at any `\b`, a scheme of any length): each start reads at most 33 characters, and each - // `://` is reached from at most 32 starts, so linear. - [/([a-z][a-z0-9+.-]{0,31}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], + // The last 1 to 32 characters of a scheme right before `://`, from any start (after `-`, `.`, `+` or a letter too: the + // updater's own rule started at any `\b`, a scheme of any length; no letter required, round 8): each start reads at + // most 33 characters, and each `://` is reached from at most 32 starts, so linear. + [/([a-z0-9+.-]{1,32}:\/\/)[^\s/@]+@/gi, (k) => `$1${k}@`, 'user info in a URL'], [/\b([A-Za-z_]*(?:token|secret|password|passwd|api[_-]?key))(\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, (k) => `$1$2${k}`, 'value after a secret name'], ]; @@ -283,6 +301,7 @@ export interface RedactTextOptions { * The prefixes of the shapes whose patterns start at `\b` (or after a character that isn't `_`): after an underscore * (`MY_ghp_...`, `x_sk-...`) a token is glued to a name, and the second pass (glued) reads it from its prefix on. */ +// check-table: never-shrink const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', 'sk-', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'xoxa-', 'xoxb-', 'xoxe', 'xoxp-', 'xoxr-', 'xoxs-', 'xapp-', 'lin_api_', 'lin_oauth_', 'AKIA', 'ASIA', 'AIza', 'eyJ', '0x', 'Bearer', 'bearer', 'BEARER', 'Basic', 'basic', 'BASIC', DASHBOARD_KEY_PREFIX]; /** * What ends in a letter or digit and may sit right before a token (#889): an escape sequence (a coloured log line, @@ -291,18 +310,17 @@ const GLUED_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_', ' * a plain `sk-`, a JSON value): the others are found wherever their prefix is. Each start reads a bounded stretch, * but a CSI's parameter run: one scan each, linear. */ -const GLUE = new RegExp( - [ - // ESC as the byte, as a JSON or C escape (`\u001b`, `\x1b`, `\033`, `\e`) or in caret notation (`^[`), then a whole - // sequence: a CSI with its parameters (`[1m`), an nF one (`(B`), or one character (`7`, `8`, `c`). - String.raw`(?:\u001b|\\u001[bB]|\\x1[bB]|\\033|\\e|\^\[)(?:\[[0-?]*[ -/]*[@-~]|[ -/]+[0-~]|[0-~])`, - // The 8-bit CSI. - String.raw`\u009b[0-?]*[ -/]*[@-~]`, - // A backslash escape in raw JSON or C-style text. - String.raw`\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])`, - ].join('|'), - 'g', -); +// check-table: never-shrink +const GLUE_PARTS = [ + // ESC as the byte, as a JSON or C escape (`\u001b`, `\x1b`, `\033`, `\e`) or in caret notation (`^[`), then a whole + // sequence: a CSI with its parameters (`[1m`), an nF one (`(B`), or one character (`7`, `8`, `c`). + String.raw`(?:\u001b|\\u001[bB]|\\x1[bB]|\\033|\\e|\^\[)(?:\[[0-?]*[ -/]*[@-~]|[ -/]+[0-~]|[0-~])`, + // The 8-bit CSI. + String.raw`\u009b[0-?]*[ -/]*[@-~]`, + // A backslash escape in raw JSON or C-style text. + String.raw`\\(?:u[0-9A-Fa-f]{4}|[nrtbf"\\/])`, +]; +const GLUE = new RegExp(GLUE_PARTS.join('|'), 'g'); /** Where a token may start glued to what is before it: after each GLUE match, and after `_` before a known prefix. */ function gluePoints(s: string): number[] { diff --git a/src/tailscale-setup.ts b/src/tailscale-setup.ts index 912ed4725..18912a876 100644 --- a/src/tailscale-setup.ts +++ b/src/tailscale-setup.ts @@ -115,9 +115,11 @@ export interface StepLine { /** * Take a key or a link out of a line before it is shown: the script never writes one, and this is the check on that. - * A key (and any other secret) by the shared redactor (#889), then the login link. + * The login link first, as on main (#889 round 8: a token glued before it, `tskey-...https://login...`, takes in its + * `https` once redacted, and the link would no longer be found), then a key and any other secret by the shared + * redactor (#889). */ -const scrub = (s: string) => redactString(s).replace(/https?:\/\/login\.tailscale\.com\S*/g, '[the login link]'); +const scrub = (s: string) => redactString(s.replace(/https?:\/\/login\.tailscale\.com\S*/g, '[the login link]')); /** Which version of the progress file this is (its size and change time; '' when there is none): it changes with every line. */ export function stepsStamp(dir = TAILSCALE_DIR): string { diff --git a/test/redact-default-union.test.ts b/test/redact-default-union.test.ts index 7be67e807..4a53e6b2a 100644 --- a/test/redact-default-union.test.ts +++ b/test/redact-default-union.test.ts @@ -87,6 +87,8 @@ const SHAPES: [string, string, string][] = (() => { ['Tailscale key', `${'tskey'}-auth-${s(30, 'Ts2Ak7Qz4Wf8')}`, s(30, 'Ts2Ak7Qz4Wf8')], ['bearer token', `Bearer ${s(36, 'Br5Tk2Nv8Zx3')}`, s(36, 'Br5Tk2Nv8Zx3')], ['URL password', `postgres://app:${s(20, 'Pw6Db3Hq9Lk1')}@db.example.com`, s(20, 'Pw6Db3Hq9Lk1')], + // A scheme longer than 32 characters whose last 32 hold no letter (#889 round 8). + ['URL password, long scheme', `x${'1'.repeat(40)}://app:${s(20, 'Lg7Sc4Hm2Pw9')}@db.example.com`, s(20, 'Lg7Sc4Hm2Pw9')], ['32-byte hex key', `0x${hex(64)}`, hex(64)], ['JSON value', `{"api_key":"${s(28, 'Js8Vk3Ap5Qy2')}"}`, s(28, 'Js8Vk3Ap5Qy2')], ['private key', `${KEY_BEGIN}\n${s(64, 'Pk1Bd7Ry4Ms9')}\n${KEY_END}`, s(64, 'Pk1Bd7Ry4Ms9')], diff --git a/test/redact-round8.test.ts b/test/redact-round8.test.ts new file mode 100644 index 000000000..a96fbe799 --- /dev/null +++ b/test/redact-round8.test.ts @@ -0,0 +1,82 @@ +// #889 round 8 (the round-7 second review): four places the shared redactor showed what it should hide. +// 1. A URL password after a scheme longer than 32 characters whose last 32 hold no letter: main's rule took a scheme +// of any length; the bounded rule needed a letter within 32 characters of `://`. +// 2. The Tailscale login link glued after a token: the link was replaced after redaction, which had taken its `https`. +// 3. A short HTTP Basic value (base64 of a short `user:password`, no digit, `+`, `/` or `=`). +// 4. The helper tables the rules read (the glued prefixes, the glue sequences, the key words) carry never-shrink marks, +// in the module and in the updater's copy, and a plain `sk-` key glued to a name is found. +import './helpers.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { redactString, secretKinds } from '../src/events/redact.js'; +import { registrationsOf } from '../src/merge-policy-weakening.js'; +import { readStepLines } from '../src/tailscale-setup.js'; +import { BRAND } from '../src/brand.js'; +import { repoRoot } from './helpers.js'; +import { run } from './token-standins.js'; + +const updaterPath = join(repoRoot, 'scripts', 'machine', `${BRAND.cli}-update.cjs`); +const updater = createRequire(import.meta.url)(updaterPath) as { redact(text: string): string }; +const broad = { broad: true } as Parameters[1]; + +test('a URL password after a long scheme whose last 32 characters hold no letter is hidden: default, broad, the updater, secretKinds', () => { + const pw = run(18, 'Pw9xSeCrEt77'); + const cases = [`x${'1'.repeat(40)}://deploy:${pw}@db.example.com/x`, `git${'-'.repeat(33)}://deploy:${pw}@db.example.com/x`, `svn+${'.'.repeat(32)}+0://u:${pw}@h/`]; + const shown: string[] = []; + for (const s of cases) { + for (const [path, out] of [['default', redactString(s)], ['broad', redactString(s, broad)], ['updater', updater.redact(s)]] as const) { + if (out.includes(pw)) shown.push(`${path}: ${s.slice(0, 40)}`); + } + if (!secretKinds(s).includes('password in a URL')) shown.push(`secretKinds: ${s.slice(0, 40)}`); + } + assert.deepEqual(shown, []); +}); + +test('the Tailscale login link glued after a token is still replaced, and the token redacted', () => { + const dir = mkdtempSync(join(tmpdir(), 'ts-steps-')); + const link = `https://login.tailscale.com/a/${'1a2b3c4d5e6f'}`; + const tokens = [`${'tskey'}-auth-${run(30, 'kAbc1DEF2CNT')}`, `${'ghp'}_${run(36)}`, `${'wld'}_${run(32, 'kQ9zX7mP2vAB')}`]; + const at = new Date().toISOString(); + writeFileSync(join(dir, 'steps.jsonl'), `${tokens.map((t) => JSON.stringify({ at, step: 'up', state: 'waiting', detail: `open ${t}${link} now` })).join('\n')}\n`); + const lines = readStepLines(dir); + assert.equal(lines.length, tokens.length); + for (const l of lines) { + assert.ok(!l.detail.includes('login.tailscale.com'), `the link is shown: ${l.detail}`); + assert.ok(!l.detail.includes('1a2b3c4d5e6f'), `the link's id is shown: ${l.detail}`); + assert.ok(l.detail.includes('[the login link]'), l.detail); + } + for (const t of tokens) assert.ok(!JSON.stringify(lines).includes(t.slice(-16)), 'the token is redacted'); +}); + +test('a short Basic value (a short user:password) is hidden on every path; Basic prose is left as it was', () => { + const pairs = [['user', 'pass'], ['admin', 'hunter'], ['bot', 'Secretpw'], ['deploy', 'abcdefghijk']].map(([u, p]) => Buffer.from(`${u}:${p}`).toString('base64')); + const shown: string[] = []; + for (const v of pairs) { + const s = `Authorization: Basic ${v}`; + for (const [path, out] of [['default', redactString(s)], ['broad', redactString(s, broad)], ['updater', updater.redact(s)]] as const) { + if (out.includes(v)) shown.push(`${path}: ${v}`); + } + } + assert.deepEqual(shown, []); + for (const prose of ['Use Basic Authentication here.', 'basic authentication', 'Basic Authorization header', 'the basic credentials flow']) assert.equal(redactString(prose), prose); +}); + +test('the helper tables carry never-shrink marks in the module and the updater copy; a plain sk- key glued to a name is found', () => { + const want = ['GLUED_PREFIXES', 'GLUE_PARTS', 'KEY_QUALIFIERS', 'CREDENTIAL_ENDINGS']; + for (const file of [join(repoRoot, 'src', 'events', 'redact.ts'), updaterPath]) { + const marked = registrationsOf(readFileSync(file, 'utf8')) + .tables.filter((t) => t.kind === 'never-shrink') + .map((t) => t.table); + assert.deepEqual( + want.filter((n) => !marked.includes(n)), + [], + file, + ); + } + const key = `${'sk'}-proj-${run(48)}`; + for (const s of [`OPENAI_KEY_${key}`, `x_${key}`]) assert.ok(!redactString(s).includes(key.slice(-20)), s); +}); diff --git a/test/token-standins.ts b/test/token-standins.ts index 5ae62b1d5..82098b2a6 100644 --- a/test/token-standins.ts +++ b/test/token-standins.ts @@ -57,6 +57,8 @@ export const SHAPES: readonly StandIn[] = [ t('API key (sk-)', 'an Anthropic API key', `sk-ant-api03-${run(93)}AA`), t('API key (sk-)', 'an Anthropic setup token', `sk-ant-oat01-${run(80)}AA`), t('API key (sk-)', 'an Anthropic admin key', `sk-ant-admin01-${run(80)}AA`), + // A plain `sk-` key (an OpenAI-style project key): no distinctive prefix, so found glued only from a glue point (#889 round 8). + t('API key (sk-)', 'a plain sk- project key', `sk-proj-${run(48)}`), // Discord: a bot token (three base64url parts), older and newer first-part lengths. t('Discord bot token', 'a Discord bot token', DISCORD), t('Discord bot token', 'a newer Discord bot token', DISCORD_NEW),