reliability-scorecard.json

{
  "schemaVersion": "coven.reliability-scorecard.v1",
  "title": "Coven reliability, recovery, and usefulness scorecard",
  "purpose": "Current decision view for issue #807. Values remain unmeasured unless retained outcome evidence exists.",
  "rows": [
    {
      "id": "journey-install-readiness",
      "category": "core_journey",
      "metric": "Install and readiness success",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "A supported packaged install verifies its binary and reaches an actionable doctor or health result.",
      "numerator": "Qualifying installs that verify and reach actionable readiness.",
      "denominator": "All qualifying install attempts in the cohort.",
      "cohort": "Supported release and platform install channels.",
      "window": "No retained installed-artifact cohort yet.",
      "source": "scripts/user-journey-e2e.mjs and setup --verify-only reports; #779 must retain the release/platform receipt.",
      "privacy": "Version, platform, stage, duration, and normalized exit class only.",
      "value": null,
      "owner": "Release maintainers",
      "target": null,
      "confidence": "Not measured; current hermetic journey coverage is binary fixture evidence, not a shipped cohort.",
      "action": "Keep unmeasured until #779 retains per-platform installed-artifact outcomes.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "journey-first-use",
      "category": "core_journey",
      "metric": "First successful session and output",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "A ready install starts a project-scoped session, accepts first input, and emits first output.",
      "numerator": "Qualifying journeys reaching first output.",
      "denominator": "Qualifying journeys that passed readiness.",
      "cohort": "Supported packaged installs using supported harnesses.",
      "window": "No retained release cohort yet.",
      "source": "scripts/user-journey-e2e.mjs and scripts/certify-release.sh.",
      "privacy": "Stage names, durations, harness id, platform, commit, and exit class; no prompt or output content.",
      "value": null,
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Not measured; deterministic fake-harness coverage cannot establish field success or latency.",
      "action": "Add stage receipts to installed-artifact certification before publishing a baseline.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "journey-control-recovery",
      "category": "core_journey",
      "metric": "Inspect, control, reconnect, and restart recovery",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "A live session can be inspected, controlled, reattached, and recovered after a supported daemon restart.",
      "numerator": "Journeys completing every required inspect, input, kill, terminal, reconnect, and restart stage.",
      "denominator": "Journeys entering the control and recovery stage.",
      "cohort": "Supported release/platform combinations in #779.",
      "window": "No retained installed-artifact recovery cohort yet.",
      "source": "user journey, daemon lifecycle, handoff/cursor, and orphan-recovery acceptance families.",
      "privacy": "Opaque synthetic session ids and normalized outcomes only.",
      "value": null,
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Not measured; lower-level acceptance exists without one retained end-to-end receipt.",
      "action": "Keep unmeasured until #779 records stage-level recovery outcomes.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "operation-launch-input",
      "category": "operation_reliability",
      "metric": "Launch and input outcome classification",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Launch and input attempts resolve to success, explicit refusal, capability mismatch, or normalized failure without silent ambiguity.",
      "numerator": "Attempts in each normalized outcome class.",
      "denominator": "All qualifying launch or input attempts.",
      "cohort": "Supported API clients and harnesses by release/platform.",
      "window": "No retained outcome cohort.",
      "source": "Daemon API contract tests and future privacy-preserving operation receipts.",
      "privacy": "Operation kind, capability/version, stage, and normalized error only.",
      "value": null,
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Not measured; contract coverage proves behavior classes, not observed rates.",
      "action": "Define a retained release-study cohort before adopting rates or thresholds.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "operation-peer-replacement",
      "category": "operation_reliability",
      "metric": "Attach, reconnect, and peer-replacement outcomes",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Attach and reconnect attempts distinguish success, stale endpoint, peer replacement, and explicit recovery-required outcomes.",
      "numerator": "Attempts in each attach or peer outcome class.",
      "denominator": "All qualifying attach or reconnect attempts.",
      "cohort": "Bundled clients against the supported local API.",
      "window": "No retained compatibility cohort.",
      "source": "coven-client health negotiation and transport tests.",
      "privacy": "Transport class and opaque peer fingerprint result only.",
      "value": null,
      "owner": "Client contract maintainers",
      "target": null,
      "confidence": "Not measured; deterministic negotiation tests do not establish incidence.",
      "action": "Populate from #779 client/platform receipts without retaining endpoint paths.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "operation-event-cursor",
      "category": "operation_reliability",
      "metric": "Ordered event consumption and cursor recovery",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Consumers retain event order and recover cursors without silently skipping required events.",
      "numerator": "Qualifying cursor journeys preserving order and recovery.",
      "denominator": "All qualifying cursor journeys.",
      "cohort": "Supported clients and release/platform combinations.",
      "window": "No retained release cohort.",
      "source": "Event cursor, handoff, and API contract acceptance tests.",
      "privacy": "Sequence numbers and normalized recovery outcomes only.",
      "value": null,
      "owner": "Client contract maintainers",
      "target": null,
      "confidence": "Not measured; acceptance coverage is not an observed shipped rate.",
      "action": "Add a cursor stage to the installed-artifact receipt before publishing a baseline.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "operation-terminal-unknown",
      "category": "operation_reliability",
      "metric": "Kill, cancel, terminal, and explicit-unknown resolution",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Control attempts resolve to a named terminal or recovery-required state; unknown is never counted as success.",
      "numerator": "Attempts in each terminal, refusal, failure, or explicit-unknown class.",
      "denominator": "All qualifying kill, cancel, and terminal-resolution attempts.",
      "cohort": "Supported harness and platform combinations.",
      "window": "No retained outcome cohort.",
      "source": "Daemon lifecycle, PTY, cancellation, and session-state acceptance tests.",
      "privacy": "Terminal class, duration, and normalized error only.",
      "value": null,
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Not measured; tests prove state semantics without incidence.",
      "action": "Preserve unknown as its own class when a release receipt is introduced.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "operation-output-cleanup-recovery",
      "category": "operation_reliability",
      "metric": "Output loss, cleanup, and crash recovery",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Output truncation, backpressure, drop, orphan, cleanup, crash, and restart outcomes are classified with recoverability.",
      "numerator": "Incidents in each loss, cleanup, orphan, and recovery class.",
      "denominator": "Qualifying sessions or injected recovery trials.",
      "cohort": "Supported release/platform combinations and declared deterministic trials.",
      "window": "No retained release cohort.",
      "source": "Output-marker, orphan-recovery, daemon lifecycle, and chaos acceptance families.",
      "privacy": "Counts, byte totals, state classes, and recovery result; no output content.",
      "value": null,
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Not measured; deterministic failure injection is supporting evidence only.",
      "action": "Keep fixture outcomes separate from shipped incidence in future receipts.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "release-exact-commit",
      "category": "release_quality",
      "metric": "Exact-commit required-check compliance",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Every published version is bound to one accepted source commit with every required check successful for that exact commit.",
      "numerator": "Published versions with a retained exact-commit governance receipt.",
      "denominator": "All published versions in the window.",
      "cohort": "GitHub Releases and package channels.",
      "window": "Begins when #805 ships the receipt.",
      "source": "#805 exact-commit release governance.",
      "privacy": "Commit, tag, check names/conclusions, signer result, and artifact digests only.",
      "value": null,
      "owner": "Release maintainers",
      "target": null,
      "confidence": "Not measured; #805 is open and no qualifying receipt exists.",
      "action": "Do not claim compliance until #805 retains the immutable receipt.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "release-channel-consistency",
      "category": "release_quality",
      "metric": "Release-channel version and support consistency",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Release, tag, registries, installers, checksums, provenance, notes, and support state identify the same public version.",
      "numerator": "Published versions with coherent channel evidence.",
      "denominator": "All published versions in the window.",
      "cohort": "Supported public release channels.",
      "window": "Begins when #805 ships reconciliation evidence.",
      "source": "#805 release receipt and reconciliation path.",
      "privacy": "Public version and artifact metadata only.",
      "value": null,
      "owner": "Release maintainers",
      "target": null,
      "confidence": "Not measured; current release metadata is not retained as one decision receipt.",
      "action": "Keep unmeasured until one receipt covers every supported channel.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "release-installed-artifacts",
      "category": "release_quality",
      "metric": "Installed-artifact E2E coverage",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Published artifacts pass the named user journey on each supported release/platform/install channel.",
      "numerator": "Certified release/platform/install-channel cells.",
      "denominator": "Required cells in the support matrix.",
      "cohort": "Published artifacts for the current supported release family.",
      "window": "Begins when #779 retains its matrix receipt.",
      "source": "#779 installed-artifact certification.",
      "privacy": "Platform, package, commit, stage outcome, duration, and normalized error only.",
      "value": null,
      "owner": "Release maintainers",
      "target": null,
      "confidence": "Not measured; #779 is open and no qualifying matrix receipt exists.",
      "action": "Do not infer artifact coverage from source-tree CI.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "release-escaped-defects",
      "category": "release_quality",
      "metric": "Escaped defects, discovery, remediation, and rollback",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Supported-release defects are classified by severity, surface, discovery source, time to safe state, and rollback or disable action.",
      "numerator": "Defects in each declared classification.",
      "denominator": "Supported releases and reporting window; rates require an approved denominator.",
      "cohort": "Supported releases only.",
      "window": "No owner-approved reporting window yet.",
      "source": "Release notes, advisories, issues, and future release-quality receipts.",
      "privacy": "Public issue/advisory metadata and normalized classifications; no private report content.",
      "value": null,
      "owner": "Release maintainers",
      "target": null,
      "confidence": "Not measured; no normalized retained ledger exists.",
      "action": "Define the window and private-to-public aggregation rules before publishing counts.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "performance-cli-fixture",
      "category": "performance_resource",
      "metric": "CLI deterministic timing collector",
      "status": "Benchmark condition/input",
      "evidenceKind": "benchmark",
      "definition": "Per-run fixture timing for command startup, daemon health, listing, tailing, and first output.",
      "numerator": "Recorded fixture durations by operation.",
      "denominator": "Configured iterations in one benchmark run.",
      "cohort": "Disposable COVEN_HOME with the fake Codex fixture on one CI host.",
      "window": "One artifact per run; no retained multi-run product window.",
      "source": "scripts/benchmark-cli.mjs.",
      "privacy": "Synthetic fixture, environment redaction, no user repository or provider content.",
      "value": "Default input is 3 iterations; report includes per-run min, median, p95, and max.",
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Valid deterministic benchmark input; insufficient for field SLOs or p95 claims.",
      "action": "Do not gate or publish as an observed product result until sample and regression budgets are approved.",
      "observedAt": null,
      "evidenceRef": "scripts/benchmark-cli.mjs"
    },
    {
      "id": "performance-chaos-fixture",
      "category": "performance_resource",
      "metric": "Concurrent chaos/resource collector",
      "status": "Benchmark condition/input",
      "evidenceKind": "benchmark",
      "definition": "Per-run deterministic load and failure-injection measurements for concurrency, cancellation, storage, backlog, memory, and recovery.",
      "numerator": "Recorded durations, rates, sizes, and resource samples by fixture trial.",
      "denominator": "Configured 1, 8, and 32-session fixture trials in one run.",
      "cohort": "Disposable local fixture environment on one CI host.",
      "window": "One report-schema-v3 artifact per run; no retained product window.",
      "source": "scripts/benchmark-chaos.mjs and its deterministic acceptance tests.",
      "privacy": "Synthetic sessions; resource totals omit process names and command lines.",
      "value": "Inputs cover 1, 8, and 32 concurrent sessions plus declared injected recovery trials.",
      "owner": "Runtime maintainers",
      "target": null,
      "confidence": "Valid benchmark condition; not field incidence or an adopted regression budget.",
      "action": "Keep deterministic acceptance separate from trend data and observed outcomes.",
      "observedAt": null,
      "evidenceRef": "scripts/benchmark-chaos.mjs"
    },
    {
      "id": "agentfs-certification",
      "category": "agentfs_security",
      "metric": "AgentFS storage and mount boundary certification",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Representative platform trials classify storage, path, materialization, concurrency, crash cleanup, credential observation, case handling, handle reuse, access control, and safe disable.",
      "numerator": "Passing certified AgentFS matrix cells.",
      "denominator": "Required cells for the declared support state.",
      "cohort": "Feature-gated AgentFS storage and experimental mount surfaces.",
      "window": "No retained certification window.",
      "source": "AgentFS conformance/mount tests and #779 certification matrix.",
      "privacy": "Synthetic paths and payloads; no credentials or repository content.",
      "value": null,
      "owner": "AgentFS maintainers",
      "target": null,
      "confidence": "Not measured; mount remains experimental and must not inherit generic CI success.",
      "action": "Keep mount disabled/experimental until its threat-derived matrix is retained.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "api-client-compatibility",
      "category": "api_compatibility",
      "metric": "API negotiation and client compatibility",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "Supported client families negotiate versions/capabilities and classify malformed, unsupported, stale-peer, and non-replay outcomes.",
      "numerator": "Compatible and explicitly denied cells by client/version/capability.",
      "denominator": "Required public contract fixture cells.",
      "cohort": "Published packages and supported client families.",
      "window": "No retained release compatibility matrix.",
      "source": "API contract fixtures, coven-client tests, and future #779 receipts.",
      "privacy": "Version, capability, route class, and normalized error only.",
      "value": null,
      "owner": "Client contract maintainers",
      "target": null,
      "confidence": "Not measured; source compatibility tests are not a published-package matrix.",
      "action": "Populate from installed published packages before reporting compatibility coverage.",
      "observedAt": null,
      "evidenceRef": null
    },
    {
      "id": "usefulness-accepted-outcome",
      "category": "usefulness",
      "metric": "Reviewed or accepted task outcome",
      "status": "Not yet measured",
      "evidenceKind": "not_measured",
      "definition": "An opt-in controlled study reaches a named reviewed or accepted outcome and records rework, rollback, elapsed time, repeat use, and a lightweight rating.",
      "numerator": "Study tasks reaching the predeclared accepted outcome.",
      "denominator": "Opt-in study tasks in the declared cohort.",
      "cohort": "Not defined; requires an accountable product decision and consent model.",
      "window": "Not defined.",
      "source": "Future opt-in beta or controlled study; never inferred from session completion.",
      "privacy": "Aggregate outcome classes and ratings only; no prompts, repository content, credentials, or full output.",
      "value": null,
      "owner": "OpenCoven product owner",
      "target": null,
      "confidence": "Not measured; no cohort, consent mechanism, or accepted-outcome contract exists.",
      "action": "Keep unmeasured until the product owner approves cohort, consent, definition, and breach action.",
      "observedAt": null,
      "evidenceRef": null
    }
  ]
}