From 4c9a3c5a643676ee13517bd1e338ceec2620adf5 Mon Sep 17 00:00:00 2001
From: Jaasiel Delgado Reyes
 <233152254+jaasieldelgado131@users.noreply.github.com>
Date: Tue, 30 Jun 2026 16:35:46 -0600
Subject: [PATCH] Add ci-failure-triage skill

---
 skills/ci-failure-triage/SKILL.md            | 112 +++++++++
 skills/ci-failure-triage/X.yaml              | 133 ++++++++++
 skills/ci-failure-triage/package.json        |  12 +
 skills/ci-failure-triage/run.mjs             | 240 +++++++++++++++++++
 skills/ci-failure-triage/tests/core.test.mjs |  64 +++++
 5 files changed, 561 insertions(+)
 create mode 100644 skills/ci-failure-triage/SKILL.md
 create mode 100644 skills/ci-failure-triage/X.yaml
 create mode 100644 skills/ci-failure-triage/package.json
 create mode 100644 skills/ci-failure-triage/run.mjs
 create mode 100644 skills/ci-failure-triage/tests/core.test.mjs

diff --git a/skills/ci-failure-triage/SKILL.md b/skills/ci-failure-triage/SKILL.md
new file mode 100644
index 000000000..b49d564ff
--- /dev/null
+++ b/skills/ci-failure-triage/SKILL.md
@@ -0,0 +1,112 @@
+---
+name: ci-failure-triage
+description: Classify CI failures as flake, infra, real-break, dep, or needs_agent and emit a read-only routing packet.
+source:
+  type: cli-tool
+  command: node
+  args:
+    - run.mjs
+runx:
+  tags:
+    - ci
+    - triage
+    - incident-response
+links:
+  source: https://github.com/jaasieldelgado131/runx
+---
+
+# CI Failure Triage
+
+## What this skill does
+
+This skill reads a bounded CI failure snapshot and emits a typed
+`runx.ci.triage.v1` packet. It classifies the failure as `flake`, `infra`,
+`real-break`, `dep`, or `unknown`, attaches cited evidence from the supplied
+logs, and returns exactly one read-only consequence when the evidence is strong
+enough:
+
+- a read-only rerun verdict for a flake,
+- a read-only operator page note for infrastructure failure, or
+- a routing decision for `issue-to-pr` when the failure is a real break or
+  dependency break.
+
+The skill never opens an issue, reruns CI, pages an operator, mints authority,
+or claims that a downstream lane has consumed its output. A downstream
+`issue-intake`, `issue-to-pr`, or `pr-review-note` run is the separate governed
+step that may act on the packet.
+
+## When to use this skill
+
+Use it at the first decision point after a CI job fails and before opening a
+tracking item. It is intended for public CI logs or explicitly authorized
+operator logs where a reviewer needs a quick, bounded classification with
+evidence citations and a conservative escalation path.
+
+## When not to use this skill
+
+Do not use it to mutate repository state, retry CI, page a human, open a bug,
+or infer root cause from thin logs. Do not feed it private secrets, tokens,
+customer data, or logs that are not authorized for the reviewer. If logs are
+truncated, contradictory, or below the configured confidence threshold, the
+skill returns `needs_agent` without a routing decision.
+
+## Inputs
+
+- `ci_failure`: object with `logs`, `commit`, and `repo_state`.
+- `repo_config`: optional repository context such as default branch and test
+  command.
+- `escalation_policy`: optional object with `min_confidence` for emitting a
+  routing decision.
+- `output_dir`: optional directory inside the skill directory for
+  `triage-packet.json` and `report.md`.
+
+## Procedure
+
+1. Normalize the CI failure, repository config, and escalation policy.
+2. Extract short evidence references from the supplied log lines.
+3. Score visible signals for `real-break`, `dep`, `infra`, and `flake`.
+4. Refuse to route when logs are absent, truncated, contradictory, or below
+   `min_confidence`.
+5. Emit the read-only triage packet with cited evidence and one bounded
+   consequence when confidence clears the threshold.
+6. Optionally write the packet and a Markdown report under `output_dir`.
+
+## Stop conditions
+
+- Missing or very short logs return `needs_agent`.
+- Truncated logs return `needs_agent`.
+- Tied evidence between failure classes returns `needs_agent`.
+- Confidence below `escalation_policy.min_confidence` returns `needs_agent`.
+- No visible evidence for the selected class returns `needs_agent`.
+
+## Output
+
+The primary output is `triage_packet`:
+
+```json
+{
+  "schema": "runx.ci.triage.v1",
+  "status": "sealed",
+  "classification": {
+    "verdict": "real-break",
+    "confidence": 0.85,
+    "evidence_refs": []
+  },
+  "triage_packet": {
+    "routing_decision": {
+      "recommended_lane": "issue-to-pr",
+      "rationale": "clear code/test failure visible in logs"
+    }
+  }
+}
+```
+
+For ambiguous failures, `status` is `needs_agent` and
+`routing_decision` is `null`.
+
+## Harness cases
+
+- `real_break_clear_logs`: clear test failure logs produce
+  `classification.verdict=real-break` and `recommended_lane=issue-to-pr`.
+- `ambiguous_truncated_logs`: truncated logs stop with `needs_agent` and no
+  routing decision.
diff --git a/skills/ci-failure-triage/X.yaml b/skills/ci-failure-triage/X.yaml
new file mode 100644
index 000000000..5af142493
--- /dev/null
+++ b/skills/ci-failure-triage/X.yaml
@@ -0,0 +1,133 @@
+skill: ci-failure-triage
+version: "0.1.0"
+
+catalog:
+  kind: graph
+  audience: public
+  visibility: public
+  role: canonical
+
+runx:
+  mutating: false
+  scopes:
+    - ci.logs.read
+    - repo.state.read
+  policy:
+    data_classification: public_ci_metadata
+    effects:
+      opens_tracking_items: false
+      reruns_ci: false
+      pages_operators: false
+      mints_authority: false
+    verifier_notes:
+      - Emits a read-only draft triage packet for downstream issue-intake, issue-to-pr, or pr-review-note.
+      - Stops at needs_agent when logs are truncated, contradictory, or below the configured confidence floor.
+  artifacts:
+    emits:
+      - triage_packet
+    wrap_as: ci_failure_triage_packet
+
+harness:
+  cases:
+    - name: real_break_clear_logs
+      runner: default
+      inputs:
+        ci_failure:
+          logs: |
+            npm test
+            FAIL tests/parser.test.ts
+            TypeError: Cannot read properties of undefined (reading 'items')
+            Expected 3 received 2
+            Tests failed in parser regression suite
+          commit: "1111111111111111111111111111111111111111"
+          repo_state:
+            branch: main
+            changed_files:
+              - src/parser.ts
+        repo_config:
+          default_branch: main
+          test_command: npm test
+        escalation_policy:
+          min_confidence: 0.75
+      expect:
+        status: sealed
+        receipt:
+          schema: runx.receipt.v1
+
+    - name: ambiguous_truncated_logs
+      runner: default
+      inputs:
+        ci_failure:
+          logs: "Error: job failed ... output truncated"
+          commit: "2222222222222222222222222222222222222222"
+          repo_state:
+            branch: main
+        repo_config:
+          default_branch: main
+        escalation_policy:
+          min_confidence: 0.75
+      expect:
+        status: needs_agent
+
+runners:
+  default:
+    default: true
+    type: graph
+    inputs:
+      ci_failure:
+        type: json
+        required: true
+        description: Object containing logs, commit, and repo_state for one CI failure.
+      repo_config:
+        type: json
+        required: false
+        description: Repository policy context such as default branch and test command.
+      escalation_policy:
+        type: json
+        required: false
+        description: Object with min_confidence threshold for emitting a routing decision.
+      output_dir:
+        type: string
+        required: false
+        description: Directory inside the skill directory where artifacts should be written.
+    graph:
+      name: ci-failure-triage
+      steps:
+        - id: classify
+          label: classify CI failure and emit triage packet
+          inputs:
+            ci_failure: "$input.ci_failure"
+            repo_config: "$input.repo_config"
+            escalation_policy: "$input.escalation_policy"
+            output_dir: "$input.output_dir"
+          run:
+            type: cli-tool
+            command: node
+            args:
+              - run.mjs
+          artifacts:
+            named_emits:
+              triage_packet: triage_packet
+            packets:
+              triage_packet: runx.ci.triage.v1
+
+        - id: human_lane
+          label: block ambiguous CI failure for agent review
+          when:
+            field: classify.triage_packet.data.status
+            equals: needs_agent
+          context:
+            reason: classify.triage_packet.data.escalation.reason
+            classification: classify.triage_packet.data.classification
+          run:
+            type: agent-task
+            agent: ci-triage-reviewer
+            task: resolve-ambiguous-ci-failure
+            outputs:
+              disposition: string
+              reviewed_classification: object
+              next_lane: object
+          instructions: >
+            Review only the supplied CI failure packet and reason. Do not infer
+            root cause, rerun CI, open issues, page operators, or mint
+            authority unless the missing evidence is supplied by the caller.
diff --git a/skills/ci-failure-triage/package.json b/skills/ci-failure-triage/package.json
new file mode 100644
index 000000000..077faf1e3
--- /dev/null
+++ b/skills/ci-failure-triage/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "runx-ci-failure-triage",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "node --test tests/*.test.mjs"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}
diff --git a/skills/ci-failure-triage/run.mjs b/skills/ci-failure-triage/run.mjs
new file mode 100644
index 000000000..71e40669c
--- /dev/null
+++ b/skills/ci-failure-triage/run.mjs
@@ -0,0 +1,240 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath } from "node:url";
+
+export const SCHEMA = "runx.ci.triage.v1";
+export const VERSION = "0.1.0";
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  const inputs = readInputs();
+  const packet = triage(inputs);
+  writeArtifacts(inputs.output_dir, packet);
+  process.stdout.write(`${JSON.stringify(packet, null, 2)}\n`);
+}
+
+export function readInputs() {
+  const raw = process.env.RUNX_INPUTS_PATH
+    ? fs.readFileSync(process.env.RUNX_INPUTS_PATH, "utf8")
+    : process.env.RUNX_INPUTS_JSON || "{}";
+  return JSON.parse(raw);
+}
+
+export function triage(inputs) {
+  const ciFailure = normalizeCiFailure(inputs.ci_failure);
+  const repoConfig = normalizeRepoConfig(inputs.repo_config);
+  const policy = normalizePolicy(inputs.escalation_policy);
+  const evidenceRefs = extractEvidence(ciFailure);
+  const signals = classifySignals(ciFailure, repoConfig);
+  const classification = decide(signals, evidenceRefs, policy);
+
+  return {
+    schema: SCHEMA,
+    version: VERSION,
+    status: classification.status,
+    classification: {
+      verdict: classification.verdict,
+      confidence: classification.confidence,
+      evidence_refs: evidenceRefs,
+    },
+    triage_packet: {
+      schema: SCHEMA,
+      status: classification.status,
+      classification: {
+        verdict: classification.verdict,
+        confidence: classification.confidence,
+        evidence_refs: evidenceRefs,
+      },
+      rerun_verdict: classification.rerun_verdict,
+      page_note: classification.page_note,
+      routing_decision: classification.routing_decision,
+      escalation: classification.escalation,
+      handoff: {
+        dispatch_by_naming: true,
+        downstream_lanes: ["issue-intake", "issue-to-pr", "pr-review-note"],
+        commencement_gate: "issue-intake",
+        effect_posture: "read-only draft routing decision",
+      },
+    },
+    policy: {
+      min_confidence: policy.min_confidence,
+      no_tracking_item_opened: true,
+      no_ci_rerun_executed: true,
+      no_operator_paged: true,
+      no_authority_minted: true,
+    },
+    observations: [
+      `classification verdict: ${classification.verdict}`,
+      `classification confidence: ${classification.confidence.toFixed(2)}`,
+      `recommended lane: ${classification.routing_decision?.recommended_lane || "none"}`,
+      `escalation reason: ${classification.escalation?.reason || "none"}`,
+      `evidence refs: ${evidenceRefs.map((ref) => ref.id).join(", ")}`,
+    ],
+    input_digest: sha256(JSON.stringify({ ci_failure: ciFailure, repo_config: repoConfig, escalation_policy: policy })),
+  };
+}
+
+function normalizeCiFailure(value) {
+  const source = value && typeof value === "object" ? value : {};
+  return {
+    logs: String(source.logs || ""),
+    commit: String(source.commit || ""),
+    repo_state: source.repo_state && typeof source.repo_state === "object" ? source.repo_state : {},
+  };
+}
+
+function normalizeRepoConfig(value) {
+  const source = value && typeof value === "object" ? value : {};
+  return {
+    default_branch: String(source.default_branch || "main"),
+    protected_paths: Array.isArray(source.protected_paths) ? source.protected_paths.map(String) : [],
+    test_command: String(source.test_command || ""),
+  };
+}
+
+function normalizePolicy(value) {
+  const source = value && typeof value === "object" ? value : {};
+  const min = Number(source.min_confidence ?? 0.75);
+  return {
+    min_confidence: Number.isFinite(min) ? Math.min(0.99, Math.max(0.5, min)) : 0.75,
+  };
+}
+
+function extractEvidence(ciFailure) {
+  const lines = ciFailure.logs.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+  return lines.slice(0, 12).map((line, index) => ({
+    id: `log:${index + 1}`,
+    quote: line.slice(0, 220),
+  }));
+}
+
+function classifySignals(ciFailure) {
+  const logs = ciFailure.logs.toLowerCase();
+  const truncated = /\btruncated\b|\.\.\.$|<snip>|output limit/i.test(ciFailure.logs);
+  const realBreakMatches = [
+    /typeerror:/i,
+    /referenceerror:/i,
+    /assertionerror/i,
+    /test(s)? failed/i,
+    /expected .* received/i,
+    /cannot find module/i,
+  ].filter((pattern) => pattern.test(ciFailure.logs)).length;
+  const depMatches = [
+    /npm err!/i,
+    /dependency/i,
+    /lockfile/i,
+    /peer dep/i,
+    /package not found/i,
+  ].filter((pattern) => pattern.test(ciFailure.logs)).length;
+  const infraMatches = [
+    /timeout/i,
+    /connection reset/i,
+    /econnreset/i,
+    /rate limit/i,
+    /runner lost/i,
+    /network/i,
+  ].filter((pattern) => pattern.test(ciFailure.logs)).length;
+  const flakeMatches = [
+    /flaky/i,
+    /rerun passed/i,
+    /intermittent/i,
+    /random seed/i,
+  ].filter((pattern) => pattern.test(ciFailure.logs)).length;
+
+  return {
+    truncated,
+    hasLogs: logs.trim().length >= 80,
+    realBreakMatches,
+    depMatches,
+    infraMatches,
+    flakeMatches,
+  };
+}
+
+function decide(signals, evidenceRefs, policy) {
+  if (!signals.hasLogs || signals.truncated || evidenceRefs.length < 2) {
+    return needsAgent("Logs are missing, truncated, or too thin to ground a CI verdict.");
+  }
+
+  const candidates = [
+    { verdict: "real-break", score: signals.realBreakMatches, lane: "issue-to-pr", rationale: "clear code/test failure visible in logs" },
+    { verdict: "dep", score: signals.depMatches, lane: "issue-to-pr", rationale: "dependency or lockfile failure visible in logs" },
+    { verdict: "infra", score: signals.infraMatches, page: "operator-page-note", rationale: "runner or network infrastructure failure visible in logs" },
+    { verdict: "flake", score: signals.flakeMatches, rerun: "read-only-rerun-verdict", rationale: "intermittent or rerun evidence visible in logs" },
+  ].sort((a, b) => b.score - a.score);
+
+  const best = candidates[0];
+  if (!best || best.score <= 0) {
+    return needsAgent("No grounded failure class is visible in the supplied logs.");
+  }
+
+  const second = candidates[1];
+  if (second && second.score > 0 && best.score - second.score <= 2) {
+    return needsAgent(`Conflicting signals for ${best.verdict} and ${second.verdict}; human triage required.`);
+  }
+
+  const confidence = Math.min(0.95, 0.62 + best.score * 0.13 + Math.min(evidenceRefs.length, 6) * 0.02);
+  if (confidence < policy.min_confidence) {
+    return needsAgent(`Grounded confidence ${confidence.toFixed(2)} is below min_confidence ${policy.min_confidence.toFixed(2)}.`);
+  }
+
+  return {
+    status: "sealed",
+    verdict: best.verdict,
+    confidence,
+    rerun_verdict: best.rerun ? { kind: best.rerun, allowed: false, rationale: best.rationale } : null,
+    page_note: best.page ? { kind: best.page, message: best.rationale, read_only: true } : null,
+    routing_decision: best.lane ? { recommended_lane: best.lane, rationale: best.rationale } : null,
+    escalation: null,
+  };
+}
+
+function needsAgent(reason) {
+  return {
+    status: "needs_agent",
+    verdict: "unknown",
+    confidence: 0,
+    rerun_verdict: null,
+    page_note: null,
+    routing_decision: null,
+    escalation: { lane: "human", reason },
+  };
+}
+
+function writeArtifacts(outputDir, packet) {
+  if (!outputDir) return;
+  const root = process.cwd();
+  const target = path.resolve(root, outputDir);
+  ensureInside(root, target);
+  fs.mkdirSync(target, { recursive: true });
+  fs.writeFileSync(path.join(target, "triage-packet.json"), `${JSON.stringify(packet, null, 2)}\n`);
+  fs.writeFileSync(path.join(target, "report.md"), renderReport(packet));
+}
+
+function renderReport(packet) {
+  return [
+    "# CI Failure Triage Report",
+    "",
+    `- Status: ${packet.status}`,
+    `- Verdict: ${packet.classification.verdict}`,
+    `- Confidence: ${packet.classification.confidence.toFixed(2)}`,
+    `- Recommended lane: ${packet.triage_packet.routing_decision?.recommended_lane || "none"}`,
+    `- Escalation: ${packet.triage_packet.escalation?.reason || "none"}`,
+    "",
+    "## Evidence",
+    "",
+    ...packet.classification.evidence_refs.map((ref) => `- ${ref.id}: ${ref.quote}`),
+    "",
+  ].join("\n");
+}
+
+function ensureInside(root, target) {
+  const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`;
+  if (target !== root && !target.startsWith(normalizedRoot)) {
+    throw new Error("output_dir must stay inside the skill directory");
+  }
+}
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
diff --git a/skills/ci-failure-triage/tests/core.test.mjs b/skills/ci-failure-triage/tests/core.test.mjs
new file mode 100644
index 000000000..91d414419
--- /dev/null
+++ b/skills/ci-failure-triage/tests/core.test.mjs
@@ -0,0 +1,64 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { triage } from "../run.mjs";
+
+test("clear real break emits issue-to-pr routing", () => {
+  const packet = triage({
+    ci_failure: {
+      commit: "1".repeat(40),
+      logs: [
+        "npm test",
+        "FAIL tests/parser.test.ts",
+        "TypeError: Cannot read properties of undefined (reading 'items')",
+        "Expected 3 received 2",
+        "Tests failed in parser regression suite",
+      ].join("\n"),
+      repo_state: { branch: "main", changed_files: ["src/parser.ts"] },
+    },
+    repo_config: { default_branch: "main", test_command: "npm test" },
+    escalation_policy: { min_confidence: 0.75 },
+  });
+
+  assert.equal(packet.status, "sealed");
+  assert.equal(packet.classification.verdict, "real-break");
+  assert.equal(packet.triage_packet.routing_decision.recommended_lane, "issue-to-pr");
+  assert.equal(packet.triage_packet.rerun_verdict, null);
+  assert.equal(packet.policy.no_tracking_item_opened, true);
+});
+
+test("ambiguous truncated logs stop without routing", () => {
+  const packet = triage({
+    ci_failure: {
+      commit: "2".repeat(40),
+      logs: "Error: job failed ... output truncated",
+      repo_state: { branch: "main" },
+    },
+    repo_config: {},
+    escalation_policy: { min_confidence: 0.75 },
+  });
+
+  assert.equal(packet.status, "needs_agent");
+  assert.equal(packet.classification.verdict, "unknown");
+  assert.equal(packet.triage_packet.routing_decision, null);
+  assert.match(packet.triage_packet.escalation.reason, /truncated|too thin/i);
+});
+
+test("conflicting signals escalate instead of guessing", () => {
+  const packet = triage({
+    ci_failure: {
+      commit: "3".repeat(40),
+      logs: [
+        "npm ERR! dependency resolution failed",
+        "TypeError: Cannot read properties of undefined",
+        "tests failed after package lockfile update",
+        "dependency graph changed in this commit",
+      ].join("\n"),
+      repo_state: { branch: "main" },
+    },
+    repo_config: {},
+    escalation_policy: { min_confidence: 0.75 },
+  });
+
+  assert.equal(packet.status, "needs_agent");
+  assert.equal(packet.triage_packet.routing_decision, null);
+});