diff --git a/README.md b/README.md index c40b3ce..f2c2560 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,45 @@ jobs: Up to **50 targets** per run. Over the cap the API refuses rather than truncating — a clean summary over a silent subset would be believed. +## Team policy — `.sato/policy.json` (v1.2.0) + +Commit a `.sato/policy.json` to the repository being checked and the diff mode +above also evaluates each changed dependency's Sato Check readings against it. +The job fails **only when a reading matches a rule you switched on**; every match +is listed in the job summary and the PR comment with the rule, what the reading +says and the evidence rule id behind it. + +```json +{ + "version": 1, + "fail_on": ["key_egress_observed", "undeclared_key_read", "unlimited_fund_action", "install_script_added", "new_host"], + "allow_hosts": ["*.base.org", "api.coingecko.com"], + "allow_subjects": ["npm:@acme/wallet"] +} +``` + +| rule | matches when | +|---|---| +| `key_egress_observed` | a planted test key was seen leaving during one of Sato Hub's runs | +| `undeclared_key_read` | code reads key material and the setup does not ask for a key | +| `unlimited_fund_action` | a fund-moving action has no configurable limit found | +| `install_script_added` | this version adds an install script | +| `new_host` | this version adds a request to a host not in `allow_hosts` (exact host or `*.suffix`) | + +`allow_subjects` exempts a subject id from every rule. `fail_on` defaults to +`["key_egress_observed"]` when omitted. + +- **`unknown` never matches a rule.** No reading is not a finding. +- The install check returns summaries, so the action reads each subject's full + profile from `GET /api/check?target=` (four at a time, 30 s timeout). A read + that fails is reported as *could not evaluate* — never a match. +- An invalid policy file is reported and ignored; it never fails the build. +- No policy file → the v1.1 behaviour: only `fail_on` (input) decides. +- New output: `policy_violations` (count of matches). + +A match names a fact your team asked to be stopped for. It is not a rating of +the dependency. + ## Outputs `verdict` (the most severe seen) · `go` · `caution` · `no` · `unknown` · diff --git a/action.yml b/action.yml index 5464961..f6ded98 100644 --- a/action.yml +++ b/action.yml @@ -87,6 +87,9 @@ outputs: key_egress_observed: description: "true when Sato Check reports a planted key seen leaving for a changed dependency." value: ${{ steps.run.outputs.key_egress_observed }} + policy_violations: + description: "Number of readings that matched a rule in the repository's .sato/policy.json (0 without a policy file)." + value: ${{ steps.run.outputs.policy_violations }} runs: using: "composite" diff --git a/src/custody.mjs b/src/custody.mjs index e7e120c..9ac25b4 100644 --- a/src/custody.mjs +++ b/src/custody.mjs @@ -13,7 +13,10 @@ import { execFileSync } from "node:child_process"; import { readFileSync, existsSync } from "node:fs"; -export const UA = "satohub-preflight-action/1.1"; +// SATO_CHECK_UA lets Sato Hub's own CI identify itself, so it is not counted as outside use. +export const UA = process.env.SATO_CHECK_UA || "satohub-preflight-action/1.2"; +export const POLICY_PATH = ".sato/policy.json"; +export const PROFILE_CONCURRENCY = 4; export const TIMEOUT_MS = 30_000; export const MARKER = ""; export const MAX_COMMAND_BYTES = 15_000; @@ -128,7 +131,7 @@ export function subjectName(sub) { } /** Markdown for the job summary and the PR comment. */ -export function renderReport(res, { changes = [], api = "https://satohub.ai", error = null, localSkills = 0 } = {}) { +export function renderReport(res, { changes = [], api = "https://satohub.ai", error = null, localSkills = 0, policy = null } = {}) { const base = api.replace(/\/$/, ""); const out = [MARKER, "## Sato Check — dependencies changed in this PR", ""]; if (changes.length) { @@ -158,6 +161,7 @@ export function renderReport(res, { changes = [], api = "https://satohub.ai", er out.push(`_Not checked: ${res.unresolved.map((u) => `\`${cell(u.input)}\` (${cell(u.reason)})`).join(", ")}_`, ""); } } + out.push(...renderPolicy(policy)); if (localSkills) out.push(`_${localSkills} skill file(s) in this repo changed. A local SKILL.md is not a registry install, so Sato Check has no record to read for it — review it in the diff._`, ""); out.push(`> ${DISCLAIMER}`, "", `[Sato Check](${base}/check)`); return out.join("\n"); @@ -177,6 +181,160 @@ export function shouldFail(res, failOnList) { return failOnList.includes("key_egress_observed") && res?.has_observed_key_egress === true; } + +// ---------- team policy (.sato/policy.json, v1.2) ---------- +// Mirrors lib/custody/policy.ts on Sato Hub (the action cannot import the app). +// A policy names which custody FACTS a team wants a build stopped for. A match +// is reported by the rule that matched and its evidence; `unknown` never matches. + +export const POLICY_RULES = ["key_egress_observed", "undeclared_key_read", "unlimited_fund_action", "install_script_added", "new_host"]; +export const RULE_TEXT = { + key_egress_observed: "a planted test key was seen leaving during a run", + undeclared_key_read: "code reads key material and the setup does not ask for a key", + unlimited_fund_action: "a fund-moving action with no configurable limit found", + install_script_added: "this version adds an install script", + new_host: "this version adds a request to a host not in allow_hosts", +}; + +const strList = (v, name) => { + if (v == null) return []; + if (!Array.isArray(v) || v.some((x) => typeof x !== "string")) return { error: `\`${name}\` must be an array of strings.` }; + return v.map((s) => s.trim()).filter(Boolean).slice(0, 500); +}; + +/** Validate a policy object. Returns {ok, policy} | {ok:false, error}. */ +export function parsePolicy(input) { + if (input == null || typeof input !== "object" || Array.isArray(input)) return { ok: false, error: "the policy must be a JSON object" }; + if (input.version !== undefined && input.version !== 1) return { ok: false, error: "only policy `version: 1` is understood" }; + const fail = input.fail_on === undefined ? ["key_egress_observed"] : strList(input.fail_on, "fail_on"); + if (!Array.isArray(fail)) return { ok: false, error: fail.error }; + for (const r of fail) if (!POLICY_RULES.includes(r)) return { ok: false, error: `unknown rule "${r}" (rules: ${POLICY_RULES.join(", ")})` }; + const hosts = strList(input.allow_hosts, "allow_hosts"); + if (!Array.isArray(hosts)) return { ok: false, error: hosts.error }; + const subjects = strList(input.allow_subjects, "allow_subjects"); + if (!Array.isArray(subjects)) return { ok: false, error: subjects.error }; + return { + ok: true, + policy: { + version: 1, + fail_on: POLICY_RULES.filter((r) => fail.includes(r)), + ...(hosts.length ? { allow_hosts: hosts.map((h) => h.toLowerCase()) } : {}), + ...(subjects.length ? { allow_subjects: subjects } : {}), + }, + }; +} + +/** Read .sato/policy.json. {policy:null} when absent; {policy:null, error} when unreadable. */ +export function readPolicy(readFile = (p) => (existsSync(p) ? readFileSync(p, "utf8") : null), path = POLICY_PATH) { + let text; + try { text = readFile(path); } catch (e) { return { policy: null, error: `could not read ${path} (${e?.message || e})` }; } + if (text == null || text === "") return { policy: null }; + let json; + try { json = JSON.parse(text); } catch { return { policy: null, error: `${path} is not valid JSON` }; } + const r = parsePolicy(json); + return r.ok ? { policy: r.policy } : { policy: null, error: `${path}: ${r.error}` }; +} + +function hostAllowed(host, allow) { + const h = String(host).toLowerCase(); + return (allow || []).some((a) => (a.startsWith("*.") ? h === a.slice(2) || h.endsWith(a.slice(1)) : h === a)); +} + +/** Which of the policy's rules this profile's readings match. Same semantics as the app. */ +export function evaluatePolicy(profile, policy) { + const on = new Set(policy.fail_on || []); + const v = []; + if (!profile || typeof profile !== "object") return { ok: true, violations: [] }; + if ((policy.allow_subjects || []).includes(profile.subject?.id)) return { ok: true, violations: [] }; + const evidence = Array.isArray(profile.evidence) ? profile.evidence : []; + if (on.has("key_egress_observed") && profile.key_egress === "observed") { + v.push({ rule: "key_egress_observed", detail: `A planted test key was sent to ${(profile.key_egress_hosts || []).join(", ") || "an outside host"}.`, evidence_rule: "O-canary-egress" }); + } + if (on.has("undeclared_key_read") && profile.key_access === "reads" && !evidence.some((e) => e.rule === "D-env-key")) { + const read = evidence.find((e) => e.rule === "T-key-read"); + v.push({ rule: "undeclared_key_read", detail: `Code reads key material and the setup does not ask for a key${read ? ` (${read.source})` : ""}.`, evidence_rule: read?.rule ?? "T-key-read" }); + } + if (on.has("unlimited_fund_action") && Array.isArray(profile.fund_actions)) { + for (const a of profile.fund_actions.filter((a) => a.limit_configurable === false)) { + v.push({ rule: "unlimited_fund_action", detail: `${a.name} (${a.action}) — no configurable limit found.`, evidence_rule: a.evidence_class === "declared" ? "D-fund-tool" : "N-unlimited-fund-action" }); + } + } + const items = Array.isArray(profile.changes?.items) ? profile.changes.items : []; + if (on.has("install_script_added")) { + for (const c of items.filter((c) => c.kind === "install_script_added")) v.push({ rule: "install_script_added", detail: c.detail, evidence_rule: "T-install-script" }); + } + if (on.has("new_host")) { + for (const c of items.filter((c) => c.kind === "host_added")) { + const host = String(c.detail || "").replace(/^adds a request to /, "").trim(); + if (hostAllowed(host, policy.allow_hosts)) continue; + const h = (profile.hosts || []).find((x) => x.host === host); + v.push({ rule: "new_host", detail: c.detail, evidence_rule: h?.evidence_class === "observed" ? "O-hosts" : "T-hosts" }); + } + } + return { ok: v.length === 0, violations: v }; +} + +/** GET /api/check?target= → the full profile. Throws on anything but a 200 with a profile. */ +export async function fetchProfile(api, id, fetchFn = fetch) { + const res = await fetchFn(`${api.replace(/\/$/, "")}/api/check?target=${encodeURIComponent(id)}`, { + headers: { "user-agent": UA, accept: "application/json" }, + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + const json = await res.json().catch(() => null); + if (res.status !== 200 || !json?.profile) throw new Error(json?.error || `HTTP ${res.status}`); + return json.profile; +} + +/** + * Evaluate every subject the install check answered for against the policy. + * The install response carries summaries only, so each profile is read from + * GET /api/check (4 at a time). A read that fails is "could not evaluate" — + * never a match. + */ +export async function evaluateSubjects(subjects, policy, { api, fetchFn = fetch, concurrency = PROFILE_CONCURRENCY } = {}) { + const violations = []; + const unevaluated = []; + const list = (subjects || []).filter((s) => s?.subject?.id && !(policy.allow_subjects || []).includes(s.subject.id)); + let next = 0; + const worker = async () => { + while (next < list.length) { + const s = list[next++]; + try { + const profile = await fetchProfile(api, s.subject.id, fetchFn); + for (const v of evaluatePolicy(profile, policy).violations) violations.push({ subject: subjectName(s), id: s.subject.id, ...v }); + } catch (e) { + unevaluated.push({ subject: subjectName(s), reason: e?.message || String(e) }); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, list.length) }, worker)); + const order = (x) => list.findIndex((s) => s.subject.id === x.id); + violations.sort((a, b) => order(a) - order(b)); + return { violations, unevaluated }; +} + +/** Markdown section for the policy outcome. */ +export function renderPolicy(policyOut) { + if (!policyOut) return []; + const out = ["**Team policy (`.sato/policy.json`)**", ""]; + if (policyOut.error) { + out.push(`_The policy file could not be used: ${cell(policyOut.error)}. No policy rule was applied; the build is not failed over it._`, ""); + return out; + } + out.push(`Rules on: ${policyOut.policy.fail_on.map((r) => `\`${r}\``).join(", ") || "none"}.`, ""); + if (policyOut.violations.length) { + out.push("| dependency | rule matched | what the reading says | evidence |", "|---|---|---|---|"); + for (const v of policyOut.violations) out.push(`| \`${cell(v.subject)}\` | \`${v.rule}\` — ${RULE_TEXT[v.rule]} | ${cell(v.detail)} | \`${v.evidence_rule}\` |`); + out.push(""); + } else { + out.push("_No reading matched a rule in the policy. An `unknown` answer never matches._", ""); + } + if (policyOut.unevaluated.length) { + out.push(`_Could not evaluate (not a match): ${policyOut.unevaluated.map((u) => `\`${cell(u.subject)}\` (${cell(u.reason)})`).join(", ")}_`, ""); + } + return out; +} + // ---------- I/O ---------- function git(args) { @@ -269,7 +427,16 @@ export async function runCustody({ api, failOn = [], token = "", env = process.e error = e?.message || String(e); log(`::warning::Sato Check was unreachable or refused (${error}). Reported as unknown; not failing the build.`); } - const report = renderReport(res, { changes, api, error, localSkills: found.skills.length }); + let policyOut = null; + const pol = (io.readPolicy || readPolicy)(); + if (pol.error) { + policyOut = { error: pol.error }; + log(`::warning::Sato Check: ${pol.error}. No policy rule applied; not failing the build over it.`); + } else if (pol.policy) { + const ev = res ? await evaluateSubjects(res.subjects, pol.policy, { api, fetchFn }) : { violations: [], unevaluated: [] }; + policyOut = { policy: pol.policy, ...ev }; + } + const report = renderReport(res, { changes, api, error, localSkills: found.skills.length, policy: policyOut }); summary(report); if (token) { @@ -286,6 +453,10 @@ export async function runCustody({ api, failOn = [], token = "", env = process.e } const egress = res?.has_observed_key_egress === true; + if (policyOut?.violations?.length) { + for (const v of policyOut.violations) log(`::error::Sato Check policy: ${v.subject} matched \`${v.rule}\` (${RULE_TEXT[v.rule]}): ${v.detail}`); + return { code: 1, report, subjects: res?.subjects?.length ?? 0, egress, violations: policyOut.violations.length }; + } if (shouldFail(res, failOn)) { log("::error::Sato Check: a planted key was seen leaving during a run for a dependency this PR adds or changes, and fail_on includes key_egress_observed. See the job summary for the host."); return { code: 1, report, subjects: res?.subjects?.length ?? 0, egress }; diff --git a/src/main.mjs b/src/main.mjs index e64c3ba..9f6b4bf 100644 --- a/src/main.mjs +++ b/src/main.mjs @@ -290,6 +290,7 @@ async function main() { if (custody.skipped) console.log(`Sato Check diff mode skipped: ${custody.skipped}.`); setOutput("custody_subjects", custody.subjects ?? 0); setOutput("key_egress_observed", custody.egress ? "true" : "false"); + setOutput("policy_violations", custody.violations ?? 0); return code || custody.code; } diff --git a/test/policy.test.mjs b/test/policy.test.mjs new file mode 100644 index 0000000..fd2fe78 --- /dev/null +++ b/test/policy.test.mjs @@ -0,0 +1,128 @@ +// Offline tests for the v1.2 team policy (.sato/policy.json). `node --test test/policy.test.mjs`. +import test from "node:test"; +import assert from "node:assert/strict"; +import { parsePolicy, readPolicy, evaluatePolicy, evaluateSubjects, renderPolicy, runCustody, UA } from "../src/custody.mjs"; + +const BANNED = /\b(safe|safely|unsafe|secure|insecure|malicious|malware|scam|trusted|trustworthy|passed|risk-free|guaranteed|audited)\b/i; + +const prof = (over = {}) => ({ + subject: { kind: "package", id: "npm:viem", name: "viem" }, + key_access: "unknown", key_egress: "unknown", key_egress_hosts: [], fund_actions: "unknown", + hosts: [], evidence: [], changes: { items: [] }, ...over, +}); +const ALL = { version: 1, fail_on: ["key_egress_observed", "undeclared_key_read", "unlimited_fund_action", "install_script_added", "new_host"] }; + +test("parsePolicy validates and normalises", () => { + assert.deepEqual(parsePolicy({}).policy.fail_on, ["key_egress_observed"]); + assert.equal(parsePolicy({ version: 2 }).ok, false); + assert.equal(parsePolicy({ fail_on: ["unsafe"] }).ok, false); + assert.equal(parsePolicy({ allow_hosts: "x" }).ok, false); + assert.equal(parsePolicy([]).ok, false); + assert.deepEqual(parsePolicy({ fail_on: ["new_host"], allow_hosts: ["API.X.com"] }).policy, { version: 1, fail_on: ["new_host"], allow_hosts: ["api.x.com"] }); +}); + +test("readPolicy: absent, invalid JSON, invalid policy", () => { + assert.deepEqual(readPolicy(() => null), { policy: null }); + assert.match(readPolicy(() => "{").error, /not valid JSON/); + assert.match(readPolicy(() => '{"fail_on":["nope"]}').error, /unknown rule/); + assert.deepEqual(readPolicy(() => '{"version":1,"fail_on":["new_host"]}').policy.fail_on, ["new_host"]); +}); + +test("unknown never matches any rule", () => { + assert.deepEqual(evaluatePolicy(prof(), ALL).violations, []); +}); + +test("each rule matches on its reading", () => { + const p = prof({ + key_egress: "observed", key_egress_hosts: ["evil.example"], key_access: "reads", + evidence: [{ rule: "T-key-read", source: "src/k.js" }], + fund_actions: [{ name: "send", action: "transfer", limit_configurable: false, evidence_class: "traced" }, { name: "swap", action: "swap", limit_configurable: true }], + changes: { items: [{ kind: "install_script_added", detail: "adds postinstall" }, { kind: "host_added", detail: "adds a request to a.example" }, { kind: "host_added", detail: "adds a request to rpc.base.org" }] }, + hosts: [{ host: "a.example", evidence_class: "observed" }], + }); + const got = evaluatePolicy(p, { ...ALL, allow_hosts: ["*.base.org"] }).violations; + assert.deepEqual(got.map((v) => v.rule), ["key_egress_observed", "undeclared_key_read", "unlimited_fund_action", "install_script_added", "new_host"]); + assert.equal(got.find((v) => v.rule === "new_host").evidence_rule, "O-hosts"); + // A declared key (D-env-key) is not undeclared. + assert.equal(evaluatePolicy(prof({ key_access: "reads", evidence: [{ rule: "D-env-key" }] }), ALL).violations.length, 0); + // Rules not switched on do not match; allow_subjects exempts. + assert.equal(evaluatePolicy(p, { version: 1, fail_on: [] }).violations.length, 0); + assert.equal(evaluatePolicy(p, { ...ALL, allow_subjects: ["npm:viem"] }).violations.length, 0); +}); + +test("evaluateSubjects reads profiles with our UA + timeout, 4 at a time, fails open", async () => { + let inFlight = 0, peak = 0; + const calls = []; + const fetchFn = async (url, init) => { + calls.push({ url, init }); + inFlight++; peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + if (url.includes("broken")) return { status: 503, json: async () => ({ error: "down" }) }; + const id = decodeURIComponent(url.split("target=")[1]); + return { status: 200, json: async () => ({ profile: prof({ subject: { id }, key_egress: "observed" }) }) }; + }; + const subjects = ["a", "b", "c", "d", "e", "broken", "skip"].map((n) => ({ subject: { id: `npm:${n}`, name: n } })); + const r = await evaluateSubjects(subjects, { ...ALL, allow_subjects: ["npm:skip"] }, { api: "https://satohub.ai/", fetchFn }); + assert.ok(peak <= 4); + assert.equal(calls.length, 6); + assert.equal(calls[0].init.headers["user-agent"], UA); + assert.ok(calls[0].init.signal); + assert.ok(calls[0].url.startsWith("https://satohub.ai/api/check?target=npm%3Aa")); + assert.deepEqual(r.violations.map((v) => v.subject), ["a", "b", "c", "d", "e"]); + assert.deepEqual(r.unevaluated.map((u) => u.subject), ["broken"]); +}); + +test("policy report describes and never judges", () => { + const md = renderPolicy({ policy: ALL, violations: [{ subject: "viem", rule: "new_host", detail: "adds a request to a.example", evidence_rule: "T-hosts" }], unevaluated: [{ subject: "x", reason: "HTTP 503" }] }).join("\n"); + assert.ok(md.includes("new_host") && md.includes("Could not evaluate (not a match)")); + assert.ok(!BANNED.test(md)); + assert.ok(!BANNED.test(renderPolicy({ error: "bad" }).join("\n"))); + assert.ok(!BANNED.test(renderPolicy({ policy: ALL, violations: [], unevaluated: [] }).join("\n"))); +}); + +// ---- end to end through runCustody ---- +const pkgBefore = JSON.stringify({ dependencies: {} }); +const pkgAfter = JSON.stringify({ dependencies: { viem: "^2.1.0" } }); +function harness({ profile, policyText, profileStatus = 200 }) { + const calls = []; + const logs = []; + const summaries = []; + const fetchFn = async (url, init = {}) => { + calls.push(url); + if (url.includes("/api/check/install")) return { status: 200, json: async () => ({ subjects: [{ subject: { id: "npm:viem", name: "viem" }, answers: {}, summary: {} }], has_observed_key_egress: false }) }; + if (url.includes("/api/check?")) return { status: profileStatus, json: async () => (profileStatus === 200 ? { profile } : { error: "down" }) }; + return { ok: true, status: 200, json: async () => [] }; + }; + const io = { + fetch: fetchFn, log: (m) => logs.push(m), summary: (m) => summaries.push(m), + readDiff: () => ({ files: ["package.json"], before: () => pkgBefore }), + readNow: () => pkgAfter, + readPolicy: () => readPolicy(() => policyText), + }; + return { calls, logs, summaries, io, env: { GITHUB_EVENT_NAME: "pull_request" } }; +} + +test("runCustody: a policy match fails the job and is listed", async () => { + const h = harness({ profile: prof({ changes: { items: [{ kind: "install_script_added", detail: "adds postinstall" }] } }), policyText: '{"version":1,"fail_on":["install_script_added"]}' }); + const r = await runCustody({ api: "https://satohub.ai", failOn: ["no"], env: h.env, io: h.io }); + assert.equal(r.code, 1); + assert.equal(r.violations, 1); + assert.ok(h.summaries[0].includes("install_script_added")); + assert.ok(h.logs.some((l) => l.startsWith("::error::Sato Check policy: viem"))); +}); + +test("runCustody: no match, unknown, unreadable profile, bad policy, no policy → exit 0", async () => { + for (const [profile, policyText, profileStatus] of [ + [prof({ key_access: "reads", evidence: [{ rule: "D-env-key" }] }), '{"fail_on":["undeclared_key_read"]}', 200], + [prof(), JSON.stringify(ALL), 200], + [prof({ key_egress: "observed" }), JSON.stringify(ALL), 503], + [prof({ key_egress: "observed" }), "{nope", 200], + ]) { + const h = harness({ profile, policyText, profileStatus }); + assert.equal((await runCustody({ api: "https://satohub.ai", env: h.env, io: h.io })).code, 0, policyText); + } + const h = harness({ profile: prof({ key_egress: "observed" }), policyText: null }); + assert.equal((await runCustody({ api: "https://satohub.ai", env: h.env, io: h.io })).code, 0); + assert.ok(!h.calls.some((u) => u.includes("/api/check?")), "no profile reads without a policy"); +});