From e00b9d20e6613a5b63f7cb9d7178dc368b9302c0 Mon Sep 17 00:00:00 2001 From: Jason Schrader Date: Wed, 27 May 2026 09:57:40 -0700 Subject: [PATCH 1/2] fix(newsdo): self-heal cost-critical signals indexes + schema-health diagnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NewsDO's hot-path `signals` composite indexes (leaderboard / correspondents- bundle / report) lived ONLY in version-gated cold-start migrations (#16, #21, #27, #28). The cold-start runner advances `migration_version` even when a statement throws (errors are caught and logged), and the code comments record this has happened before ("migration 10 failed silently on production"). A silently-failed index migration is never retried, so the index can go permanently missing in production while the schema looks complete — the same class of bug that dropped landing-page's inbox_messages indexes and drove a multi-billion-row/day full-scan D1 bill. NewsDO is currently the account's top rows-read surface (~1.8B/day). This moves the at-risk signals indexes into the always-re-applied base SCHEMA_SQL so they self-heal on every cold start (idempotent CREATE INDEX IF NOT EXISTS; all referenced columns exist in the base signals table). Adds GET /api/config/schema-health (public, read-only): diffs the live sqlite_master against EXPECTED_SIGNALS_INDEXES and reports any missing index + signals row count. DO-embedded SQLite has no external `wrangler d1 insights` equivalent, and DO console output does not reach worker-logs (that's *why* the failures were silent), so an on-demand endpoint is the reliable drift detector. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/lib/do-client.ts | 16 ++++++++++++++++ src/objects/news-do.ts | 36 +++++++++++++++++++++++++++++++++++- src/objects/schema.ts | 36 ++++++++++++++++++++++++++++++++++++ src/routes/config.ts | 11 ++++++++++- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/lib/do-client.ts b/src/lib/do-client.ts index a80320b..a6aaba2 100644 --- a/src/lib/do-client.ts +++ b/src/lib/do-client.ts @@ -911,6 +911,22 @@ export async function setConfig(env: Env, key: string, value: string): Promise { + const stub = getStub(env); + const result = await doFetch(stub, "/schema-health"); + if (!result.ok) throw new Error(result.error ?? "Failed to read schema health"); + if (result.data === undefined) throw new Error("Missing data in response"); + return result.data; +} + // --------------------------------------------------------------------------- // Signal Review (Publisher editorial actions) // --------------------------------------------------------------------------- diff --git a/src/objects/news-do.ts b/src/objects/news-do.ts index 81ad21e..e65139e 100644 --- a/src/objects/news-do.ts +++ b/src/objects/news-do.ts @@ -5,7 +5,7 @@ import type { Env, Beat, Signal, SignalStatus, Streak, Brief, Classified, Classi import { validateSlug, validateHexColor, sanitizeString, validateDateFormat } from "../lib/validators"; import { generateId, getUTCDate, getUTCYesterday, getUTCDayStart, getUTCDayEnd, getNextDate } from "../lib/helpers"; import { CLASSIFIED_DURATION_DAYS, CLASSIFIED_BRIEF_SLOTS, CLASSIFIED_BRIEF_MAX_CHARS, CLASSIFIED_STATUSES, SIGNAL_COOLDOWN_HOURS, BEAT_EXPIRY_DAYS, MAX_SIGNALS_PER_DAY, MAX_INCLUDED_SIGNALS_PER_BRIEF, MAX_APPROVED_SIGNALS_PER_DAY, SIGNAL_STATUSES, REVIEWABLE_SIGNAL_STATUSES, CONFIG_PUBLISHER_ADDRESS, BRIEF_INCLUSION_PAYOUT_SATS, WEEKLY_PRIZE_1ST_SATS, WEEKLY_PRIZE_2ND_SATS, WEEKLY_PRIZE_3RD_SATS, SCORING_WEIGHTS, PAYMENT_STAGE_TTL_MS, PENDING_PAYMENT_STATUS } from "../lib/constants"; -import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL } from "./schema"; +import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL, MIGRATION_BEAT_CONSOLIDATION_SQL, MIGRATION_SIGNAL_SCORING_SQL, MIGRATION_APR7_EARNINGS_SQL, MIGRATION_CLASSIFIEDS_TXID_UNIQUE_SQL, MIGRATION_SIGNAL_HOT_PATH_INDEXES_SQL, MIGRATION_CORRESPONDENTS_BUNDLE_INDEXES_SQL, MIGRATION_CORRESPONDENT_STATS_SQL, MIGRATION_SIGNAL_PAYMENT_SQL, EXPECTED_SIGNALS_INDEXES } from "./schema"; import { scoreSignal } from "../lib/signal-scorer"; // ── State machine transition maps ── @@ -3755,6 +3755,40 @@ export class NewsDO extends DurableObject { }); }); + // GET /schema-health — read-only schema-drift report. Diffs the live + // sqlite_master against EXPECTED_SIGNALS_INDEXES so a silently-dropped + // index (the version-gated-migration failure mode) surfaces on demand + // instead of quietly becoming a full-table scan. No external D1-style + // `insights` exists for DO SQLite, so this endpoint is the equivalent. + this.router.get("/schema-health", (c) => { + const indexRows = this.ctx.storage.sql + .exec( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%' ORDER BY name" + ) + .toArray() as { name: string }[]; + const liveIndexes = indexRows.map((r) => r.name); + const liveSet = new Set(liveIndexes); + const missingSignalsIndexes = EXPECTED_SIGNALS_INDEXES.filter( + (name) => !liveSet.has(name) + ); + + const countRows = this.ctx.storage.sql + .exec("SELECT COUNT(*) as count FROM signals") + .toArray(); + const signalsRowCount = (countRows[0] as { count: number }).count; + + return c.json({ + ok: true, + data: { + healthy: missingSignalsIndexes.length === 0, + missing_signals_indexes: missingSignalsIndexes, + signals_row_count: signalsRowCount, + live_index_count: liveIndexes.length, + live_indexes: liveIndexes, + }, + }); + }); + // GET /correspondents — agents with signal counts, last active. // Reads from materialised correspondent_stats (~430 rows) instead of // grouping over the full signals table. diff --git a/src/objects/schema.ts b/src/objects/schema.ts index 1abb0bf..bc8956f 100644 --- a/src/objects/schema.ts +++ b/src/objects/schema.ts @@ -137,6 +137,18 @@ CREATE INDEX IF NOT EXISTS idx_signals_status_created ON signals(status, creat CREATE INDEX IF NOT EXISTS idx_signals_status_btc_created ON signals(status, btc_address, created_at DESC); CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed_created ON signals(status, reviewed_at DESC, created_at DESC); CREATE INDEX IF NOT EXISTS idx_signals_correction_of ON signals(correction_of); +-- Hot-path composite indexes for the leaderboard / correspondents-bundle / report +-- queries. These previously lived ONLY in version-gated cold-start migrations +-- (#16, #21, #27, #28). A migration that fails silently is never retried (the +-- version counter advances regardless), so the index can go permanently missing +-- in production while the code believes the schema is complete — the same class +-- of bug that dropped landing-page's inbox indexes. Keeping them here in the +-- always-re-applied base schema makes them self-heal on every cold start. +CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed ON signals(status, reviewed_at); +CREATE INDEX IF NOT EXISTS idx_signals_correction_created ON signals(correction_of, created_at); +CREATE INDEX IF NOT EXISTS idx_signals_correction_btc_created ON signals(correction_of, btc_address, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_signals_beat_btc_correction_created ON signals(beat_slug, btc_address, correction_of, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_signals_quality_score ON signals(quality_score); CREATE INDEX IF NOT EXISTS idx_earnings_btc_address ON earnings(btc_address); CREATE INDEX IF NOT EXISTS idx_classifieds_btc_address ON classifieds(btc_address); CREATE INDEX IF NOT EXISTS idx_classifieds_expires_at ON classifieds(expires_at); @@ -150,6 +162,30 @@ CREATE INDEX IF NOT EXISTS idx_referral_recruit ON referral_credits(recr CREATE INDEX IF NOT EXISTS idx_payment_staging_status ON payment_staging(stage_status); `; +/** + * Cost-critical `signals` indexes that MUST exist in production. The + * GET /api/config/schema-health endpoint diffs the live `sqlite_master` + * against this set so a silently-dropped index surfaces on demand instead of + * quietly turning every leaderboard/correspondents read into a full-table scan. + * Keep in sync with the signals indexes in SCHEMA_SQL above. + */ +export const EXPECTED_SIGNALS_INDEXES: readonly string[] = [ + "idx_signals_beat_slug", + "idx_signals_beat_created", + "idx_signals_btc_address", + "idx_signals_btc_created", + "idx_signals_created_at", + "idx_signals_status_created", + "idx_signals_status_btc_created", + "idx_signals_status_reviewed_created", + "idx_signals_correction_of", + "idx_signals_status_reviewed", + "idx_signals_correction_created", + "idx_signals_correction_btc_created", + "idx_signals_beat_btc_correction_created", + "idx_signals_quality_score", +]; + /** * Migration SQL for existing databases that lack Phase 0 columns. * Each statement is wrapped in a try/catch-friendly pattern (columns may already exist). diff --git a/src/routes/config.ts b/src/routes/config.ts index 9331913..93bf298 100644 --- a/src/routes/config.ts +++ b/src/routes/config.ts @@ -7,7 +7,7 @@ import { Hono } from "hono"; import type { Env, AppVariables } from "../lib/types"; -import { getConfig, setConfig } from "../lib/do-client"; +import { getConfig, setConfig, getSchemaHealth } from "../lib/do-client"; import { validateBtcAddress } from "../lib/validators"; import { verifyAuth } from "../services/auth"; import { CONFIG_PUBLISHER_ADDRESS, PARENT_INSCRIPTION_ID } from "../lib/constants"; @@ -26,6 +26,15 @@ configRouter.get("/api/config/publisher", async (c) => { }); }); +// GET /api/config/schema-health — read-only schema-drift report (public). +// Diffs the live NewsDO sqlite_master against the cost-critical signals index +// set so a silently-dropped index surfaces on demand. Returns healthy:false + +// the missing index names if any are absent. +configRouter.get("/api/config/schema-health", async (c) => { + const health = await getSchemaHealth(c.env); + return c.json(health, health.healthy ? 200 : 503); +}); + // POST /api/config/publisher — designate a Publisher (BIP-322 auth required) // If no Publisher is set, any BIP-322-authenticated agent can claim it (bootstrap). // Once set, only the current Publisher can re-designate. From 70689a4a4e4fa05b1ac4499be13e63144f9b6d75 Mon Sep 17 00:00:00 2001 From: Jason Schrader Date: Wed, 27 May 2026 11:51:47 -0700 Subject: [PATCH 2/2] =?UTF-8?q?fix(newsdo):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20drop=20quality=5Fscore=20base=20index,=20opt-in=20count,=20t?= =?UTF-8?q?ests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P1 (Codex/Copilot): remove idx_signals_quality_score from base SCHEMA_SQL. SCHEMA_SQL runs before versioned migrations; quality_score is added by migration #24's ALTER. On a DO where that ALTER silently failed, a base-schema index on the column would throw no-such-column and brick DO construction. The index stays in migration #24 where the column is guaranteed to exist; removed from EXPECTED_SIGNALS_INDEXES. The other 4 indexes reference columns already used by pre-existing base indexes, so they provably exist live. - COUNT(*) (Copilot/arc0btc): signals_row_count is now opt-in via ?include_count=true. The default public/unthrottled health check does no full-table scan; signals_row_count is null unless requested. - Tests (Copilot): add /api/config/schema-health coverage asserting healthy + every EXPECTED_SIGNALS_INDEXES present (guards against EXPECTED drift) and the default-vs-include_count behavior. - arc0btc: typed intermediate for the count row; TODO to extend self-heal + EXPECTED set to claims/earnings once /schema-health confirms their risk. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/__tests__/schema-migration.test.ts | 39 ++++++++++++++++++++++++++ src/lib/do-client.ts | 13 +++++++-- src/objects/news-do.ts | 15 +++++++--- src/objects/schema.ts | 19 +++++++++++-- src/routes/config.ts | 4 ++- 5 files changed, 79 insertions(+), 11 deletions(-) diff --git a/src/__tests__/schema-migration.test.ts b/src/__tests__/schema-migration.test.ts index a5608ed..d0344c4 100644 --- a/src/__tests__/schema-migration.test.ts +++ b/src/__tests__/schema-migration.test.ts @@ -1,5 +1,14 @@ import { describe, it, expect } from "vitest"; import { SELF } from "cloudflare:test"; +import { EXPECTED_SIGNALS_INDEXES } from "../objects/schema"; + +interface SchemaHealthBody { + healthy: boolean; + missing_signals_indexes: string[]; + signals_row_count: number | null; + live_index_count: number; + live_indexes: string[]; +} /** * Migration-path tests that verify the DO constructor correctly runs @@ -81,3 +90,33 @@ describe("DO constructor: schema initialization", () => { expect(slugs).not.toContain("dev-tools"); }); }); + +describe("GET /api/config/schema-health", () => { + it("reports healthy with every expected signals index present on a fresh DB", async () => { + const res = await SELF.fetch("http://example.com/api/config/schema-health"); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.healthy).toBe(true); + expect(body.missing_signals_indexes).toEqual([]); + // Guards against the EXPECTED set drifting from what SCHEMA_SQL creates: + // every expected index must actually exist in the live DB. + for (const idx of EXPECTED_SIGNALS_INDEXES) { + expect(body.live_indexes).toContain(idx); + } + expect(body.live_index_count).toBe(body.live_indexes.length); + }); + + it("omits the COUNT(*) scan by default, includes it on ?include_count=true", async () => { + const noCount = await ( + await SELF.fetch("http://example.com/api/config/schema-health") + ).json(); + expect(noCount.signals_row_count).toBeNull(); + + const withCount = await ( + await SELF.fetch( + "http://example.com/api/config/schema-health?include_count=true" + ) + ).json(); + expect(typeof withCount.signals_row_count).toBe("number"); + }); +}); diff --git a/src/lib/do-client.ts b/src/lib/do-client.ts index a6aaba2..ec665bd 100644 --- a/src/lib/do-client.ts +++ b/src/lib/do-client.ts @@ -914,14 +914,21 @@ export async function setConfig(env: Env, key: string, value: string): Promise { +export async function getSchemaHealth( + env: Env, + includeCount = false +): Promise { const stub = getStub(env); - const result = await doFetch(stub, "/schema-health"); + const result = await doFetch( + stub, + `/schema-health${includeCount ? "?include_count=true" : ""}` + ); if (!result.ok) throw new Error(result.error ?? "Failed to read schema health"); if (result.data === undefined) throw new Error("Missing data in response"); return result.data; diff --git a/src/objects/news-do.ts b/src/objects/news-do.ts index e65139e..0ab7ab3 100644 --- a/src/objects/news-do.ts +++ b/src/objects/news-do.ts @@ -3772,10 +3772,17 @@ export class NewsDO extends DurableObject { (name) => !liveSet.has(name) ); - const countRows = this.ctx.storage.sql - .exec("SELECT COUNT(*) as count FROM signals") - .toArray(); - const signalsRowCount = (countRows[0] as { count: number }).count; + // `signals_row_count` is opt-in: COUNT(*) full-scans the signals table, + // and this endpoint is public + unthrottled. Default off so routine / + // probing health checks add no rows-read; callers that want the count + // pass ?include_count=true. + let signalsRowCount: number | null = null; + if (c.req.query("include_count") === "true") { + const countRow = this.ctx.storage.sql + .exec("SELECT COUNT(*) as count FROM signals") + .toArray()[0] as { count: number }; + signalsRowCount = countRow.count; + } return c.json({ ok: true, diff --git a/src/objects/schema.ts b/src/objects/schema.ts index bc8956f..228eb46 100644 --- a/src/objects/schema.ts +++ b/src/objects/schema.ts @@ -139,16 +139,27 @@ CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed_created ON signals(status CREATE INDEX IF NOT EXISTS idx_signals_correction_of ON signals(correction_of); -- Hot-path composite indexes for the leaderboard / correspondents-bundle / report -- queries. These previously lived ONLY in version-gated cold-start migrations --- (#16, #21, #27, #28). A migration that fails silently is never retried (the +-- (#16, #21, #28). A migration that fails silently is never retried (the -- version counter advances regardless), so the index can go permanently missing -- in production while the code believes the schema is complete — the same class -- of bug that dropped landing-page's inbox indexes. Keeping them here in the -- always-re-applied base schema makes them self-heal on every cold start. +-- +-- IMPORTANT: SCHEMA_SQL runs BEFORE the versioned migrations, so only columns +-- present in the base signals CREATE TABLE above may be indexed here. Every +-- column below (status, reviewed_at, correction_of, beat_slug, btc_address, +-- created_at) is already referenced by a base-schema index, so it provably +-- exists on the live DB. quality_score is deliberately NOT indexed here: it is +-- added by versioned migration #24, and if that ALTER had silently failed, a +-- base-schema index on it would throw a no-such-column error and brick DO +-- construction before any migration could repair it. Its index stays in +-- migration #24, where the column is guaranteed to exist. +-- TODO: extend this self-heal + EXPECTED set to claims/earnings indexes once +-- /schema-health confirms their version-gated indexes are also at risk. CREATE INDEX IF NOT EXISTS idx_signals_status_reviewed ON signals(status, reviewed_at); CREATE INDEX IF NOT EXISTS idx_signals_correction_created ON signals(correction_of, created_at); CREATE INDEX IF NOT EXISTS idx_signals_correction_btc_created ON signals(correction_of, btc_address, created_at DESC); CREATE INDEX IF NOT EXISTS idx_signals_beat_btc_correction_created ON signals(beat_slug, btc_address, correction_of, created_at DESC); -CREATE INDEX IF NOT EXISTS idx_signals_quality_score ON signals(quality_score); CREATE INDEX IF NOT EXISTS idx_earnings_btc_address ON earnings(btc_address); CREATE INDEX IF NOT EXISTS idx_classifieds_btc_address ON classifieds(btc_address); CREATE INDEX IF NOT EXISTS idx_classifieds_expires_at ON classifieds(expires_at); @@ -183,7 +194,9 @@ export const EXPECTED_SIGNALS_INDEXES: readonly string[] = [ "idx_signals_correction_created", "idx_signals_correction_btc_created", "idx_signals_beat_btc_correction_created", - "idx_signals_quality_score", + // NB: idx_signals_quality_score is intentionally excluded — it stays in + // versioned migration #24 because base SCHEMA_SQL runs before that column's + // ALTER (see the SCHEMA_SQL comment above). ]; /** diff --git a/src/routes/config.ts b/src/routes/config.ts index 93bf298..39a8ced 100644 --- a/src/routes/config.ts +++ b/src/routes/config.ts @@ -31,7 +31,9 @@ configRouter.get("/api/config/publisher", async (c) => { // set so a silently-dropped index surfaces on demand. Returns healthy:false + // the missing index names if any are absent. configRouter.get("/api/config/schema-health", async (c) => { - const health = await getSchemaHealth(c.env); + // include_count is opt-in: COUNT(*) full-scans signals, and this route is public. + const includeCount = c.req.query("include_count") === "true"; + const health = await getSchemaHealth(c.env, includeCount); return c.json(health, health.healthy ? 200 : 503); });