fix(newsdo): self-heal cost-critical signals indexes + schema-health diagnostic

src/__tests__/schema-migration.test.ts

@@ -1,5 +1,14 @@
 import { describe, it, expect } from "vitest";
 import { SELF } from "cloudflare:test";
+import { EXPECTED_SIGNALS_INDEXES } from "../objects/schema";
+
+interface SchemaHealthBody {
+  healthy: boolean;
+  missing_signals_indexes: string[];
+  signals_row_count: number | null;
+  live_index_count: number;
+  live_indexes: string[];
+}
 
 /**
  * Migration-path tests that verify the DO constructor correctly runs
@@ -81,3 +90,33 @@ describe("DO constructor: schema initialization", () => {
     expect(slugs).not.toContain("dev-tools");
   });
 });
+
+describe("GET /api/config/schema-health", () => {
+  it("reports healthy with every expected signals index present on a fresh DB", async () => {
+    const res = await SELF.fetch("http://example.com/api/config/schema-health");
+    expect(res.status).toBe(200);
+    const body = await res.json<SchemaHealthBody>();
+    expect(body.healthy).toBe(true);
+    expect(body.missing_signals_indexes).toEqual([]);
+    // Guards against the EXPECTED set drifting from what SCHEMA_SQL creates:
+    // every expected index must actually exist in the live DB.
+    for (const idx of EXPECTED_SIGNALS_INDEXES) {
+      expect(body.live_indexes).toContain(idx);
+    }
+    expect(body.live_index_count).toBe(body.live_indexes.length);
+  });
+
+  it("omits the COUNT(*) scan by default, includes it on ?include_count=true", async () => {
+    const noCount = await (
+      await SELF.fetch("http://example.com/api/config/schema-health")
+    ).json<SchemaHealthBody>();
+    expect(noCount.signals_row_count).toBeNull();
+
+    const withCount = await (
+      await SELF.fetch(
+        "http://example.com/api/config/schema-health?include_count=true"
+      )
+    ).json<SchemaHealthBody>();
+    expect(typeof withCount.signals_row_count).toBe("number");
+  });
+});

src/lib/do-client.ts

@@ -911,6 +911,29 @@ export async function setConfig(env: Env, key: string, value: string): Promise<D
   });
 }
 
+export interface SchemaHealth {
+  healthy: boolean;
+  missing_signals_indexes: string[];
+  /** null unless the caller requested includeCount (COUNT(*) full-scans signals). */
+  signals_row_count: number | null;
+  live_index_count: number;
+  live_indexes: string[];
+}
+
+export async function getSchemaHealth(
+  env: Env,
+  includeCount = false
+): Promise<SchemaHealth> {
+  const stub = getStub(env);
+  const result = await doFetch<SchemaHealth>(
+    stub,
+    `/schema-health${includeCount ? "?include_count=true" : ""}`
+  );
+  if (!result.ok) throw new Error(result.error ?? "Failed to read schema health");
+  if (result.data === undefined) throw new Error("Missing data in response");
+  return result.data;
+}
+
 // ---------------------------------------------------------------------------
 // Signal Review (Publisher editorial actions)
 // ---------------------------------------------------------------------------

src/objects/news-do.ts

@@ -5,7 +5,7 @@ import type { Env, Beat, Signal, SignalStatus, Streak, Brief, Classified, Classi
 import { validateSlug, validateHexColor, sanitizeString, validateDateFormat } from "../lib/validators";
 import { generateId, getUTCDate, getUTCYesterday, getUTCDayStart, getUTCDayEnd, getNextDate } from "../lib/helpers";
 import { CLASSIFIED_DURATION_DAYS, CLASSIFIED_BRIEF_SLOTS, CLASSIFIED_BRIEF_MAX_CHARS, CLASSIFIED_STATUSES, SIGNAL_COOLDOWN_HOURS, BEAT_EXPIRY_DAYS, MAX_SIGNALS_PER_DAY, MAX_INCLUDED_SIGNALS_PER_BRIEF, MAX_APPROVED_SIGNALS_PER_DAY, SIGNAL_STATUSES, REVIEWABLE_SIGNAL_STATUSES, CONFIG_PUBLISHER_ADDRESS, BRIEF_INCLUSION_PAYOUT_SATS, WEEKLY_PRIZE_1ST_SATS, WEEKLY_PRIZE_2ND_SATS, WEEKLY_PRIZE_3RD_SATS, SCORING_WEIGHTS, PAYMENT_STAGE_TTL_MS, PENDING_PAYMENT_STATUS } from "../lib/constants";
-import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL } from "./schema";
+import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL, EXPECTED_SIGNALS_INDEXES } from "./schema";
 import { scoreSignal } from "../lib/signal-scorer";
 
 // ── State machine transition maps ──
@@ -3755,6 +3755,47 @@ export class NewsDO extends DurableObject<Env> {
       });
     });
 
+    // GET /schema-health — read-only schema-drift report. Diffs the live
+    // sqlite_master against EXPECTED_SIGNALS_INDEXES so a silently-dropped
+    // index (the version-gated-migration failure mode) surfaces on demand
+    // instead of quietly becoming a full-table scan. No external D1-style
+    // `insights` exists for DO SQLite, so this endpoint is the equivalent.
+    this.router.get("/schema-health", (c) => {
+      const indexRows = this.ctx.storage.sql
+        .exec(
+          "SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%' ORDER BY name"
+        )
+        .toArray() as { name: string }[];
+      const liveIndexes = indexRows.map((r) => r.name);
+      const liveSet = new Set(liveIndexes);
+      const missingSignalsIndexes = EXPECTED_SIGNALS_INDEXES.filter(
+        (name) => !liveSet.has(name)
+      );
+
+      // `signals_row_count` is opt-in: COUNT(*) full-scans the signals table,
+      // and this endpoint is public + unthrottled. Default off so routine /
+      // probing health checks add no rows-read; callers that want the count
+      // pass ?include_count=true.
+      let signalsRowCount: number | null = null;
+      if (c.req.query("include_count") === "true") {
+        const countRow = this.ctx.storage.sql
+          .exec("SELECT COUNT(*) as count FROM signals")
+          .toArray()[0] as { count: number };
+        signalsRowCount = countRow.count;
+      }
+
+      return c.json({
+        ok: true,
+        data: {
+          healthy: missingSignalsIndexes.length === 0,
+          missing_signals_indexes: missingSignalsIndexes,
+          signals_row_count: signalsRowCount,
+          live_index_count: liveIndexes.length,
+          live_indexes: liveIndexes,
+        },
+      });
+    });
+
     // GET /correspondents — agents with signal counts, last active.
     // Reads from materialised correspondent_stats (~430 rows) instead of
     // grouping over the full signals table.

src/objects/schema.ts

@@ -137,6 +137,29 @@ CREATE INDEX IF NOT EXISTS idx_signals_status_created   ON signals(status, creat
 CREATE INDEX IF NOT EXISTS idx_signals_status_btc_created ON signals(status, btc_address, created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed_created ON signals(status, reviewed_at DESC, created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_signals_correction_of    ON signals(correction_of);
+-- Hot-path composite indexes for the leaderboard / correspondents-bundle / report
+-- queries. These previously lived ONLY in version-gated cold-start migrations
+-- (#16, #21, #28). A migration that fails silently is never retried (the
+-- version counter advances regardless), so the index can go permanently missing
+-- in production while the code believes the schema is complete — the same class
+-- of bug that dropped landing-page's inbox indexes. Keeping them here in the
+-- always-re-applied base schema makes them self-heal on every cold start.
+--
+-- IMPORTANT: SCHEMA_SQL runs BEFORE the versioned migrations, so only columns
+-- present in the base signals CREATE TABLE above may be indexed here. Every
+-- column below (status, reviewed_at, correction_of, beat_slug, btc_address,
+-- created_at) is already referenced by a base-schema index, so it provably
+-- exists on the live DB. quality_score is deliberately NOT indexed here: it is
+-- added by versioned migration #24, and if that ALTER had silently failed, a
+-- base-schema index on it would throw a no-such-column error and brick DO
+-- construction before any migration could repair it. Its index stays in
+-- migration #24, where the column is guaranteed to exist.
+-- TODO: extend this self-heal + EXPECTED set to claims/earnings indexes once
+-- /schema-health confirms their version-gated indexes are also at risk.
+CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed          ON signals(status, reviewed_at);
+CREATE INDEX IF NOT EXISTS idx_signals_correction_created       ON signals(correction_of, created_at);
+CREATE INDEX IF NOT EXISTS idx_signals_correction_btc_created   ON signals(correction_of, btc_address, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_signals_beat_btc_correction_created ON signals(beat_slug, btc_address, correction_of, created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_earnings_btc_address     ON earnings(btc_address);
 CREATE INDEX IF NOT EXISTS idx_classifieds_btc_address  ON classifieds(btc_address);
 CREATE INDEX IF NOT EXISTS idx_classifieds_expires_at   ON classifieds(expires_at);
@@ -150,6 +173,32 @@ CREATE INDEX IF NOT EXISTS idx_referral_recruit         ON referral_credits(recr
 CREATE INDEX IF NOT EXISTS idx_payment_staging_status   ON payment_staging(stage_status);
 `;
 
+/**
+ * Cost-critical `signals` indexes that MUST exist in production. The
+ * GET /api/config/schema-health endpoint diffs the live `sqlite_master`
+ * against this set so a silently-dropped index surfaces on demand instead of
+ * quietly turning every leaderboard/correspondents read into a full-table scan.
+ * Keep in sync with the signals indexes in SCHEMA_SQL above.
+ */
+export const EXPECTED_SIGNALS_INDEXES: readonly string[] = [
+  "idx_signals_beat_slug",
+  "idx_signals_beat_created",
+  "idx_signals_btc_address",
+  "idx_signals_btc_created",
+  "idx_signals_created_at",
+  "idx_signals_status_created",
+  "idx_signals_status_btc_created",
+  "idx_signals_status_reviewed_created",
+  "idx_signals_correction_of",
+  "idx_signals_status_reviewed",
+  "idx_signals_correction_created",
+  "idx_signals_correction_btc_created",
+  "idx_signals_beat_btc_correction_created",
+  // NB: idx_signals_quality_score is intentionally excluded — it stays in
+  // versioned migration #24 because base SCHEMA_SQL runs before that column's
+  // ALTER (see the SCHEMA_SQL comment above).
+];
+
 /**
  * Migration SQL for existing databases that lack Phase 0 columns.
  * Each statement is wrapped in a try/catch-friendly pattern (columns may already exist).

src/routes/config.ts

@@ -7,7 +7,7 @@
 
 import { Hono } from "hono";
 import type { Env, AppVariables } from "../lib/types";
-import { getConfig, setConfig } from "../lib/do-client";
+import { getConfig, setConfig, getSchemaHealth } from "../lib/do-client";
 import { validateBtcAddress } from "../lib/validators";
 import { verifyAuth } from "../services/auth";
 import { CONFIG_PUBLISHER_ADDRESS, PARENT_INSCRIPTION_ID } from "../lib/constants";
@@ -26,6 +26,17 @@ configRouter.get("/api/config/publisher", async (c) => {
   });
 });
 
+// GET /api/config/schema-health — read-only schema-drift report (public).
+// Diffs the live NewsDO sqlite_master against the cost-critical signals index
+// set so a silently-dropped index surfaces on demand. Returns healthy:false +
+// the missing index names if any are absent.
+configRouter.get("/api/config/schema-health", async (c) => {
+  // include_count is opt-in: COUNT(*) full-scans signals, and this route is public.
+  const includeCount = c.req.query("include_count") === "true";
+  const health = await getSchemaHealth(c.env, includeCount);
+  return c.json(health, health.healthy ? 200 : 503);
+});
+
 // POST /api/config/publisher — designate a Publisher (BIP-322 auth required)
 // If no Publisher is set, any BIP-322-authenticated agent can claim it (bootstrap).
 // Once set, only the current Publisher can re-designate.