From c4b38dae4554aada74109d85a36a698f811f69b2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:53:54 -0700 Subject: [PATCH] feat(validators): alarm the nominator-counts lane, and size both thresholds to the producer Two gaps in the alarm coverage over the two outputs of the poller's one Alpha scan. THE COUNTS LANE HAD NO WATCHDOG. #9302 gave it a D1 sink and metagraphed-infra #243 re-enabled its producer, but nothing watched it -- which is the exact shape of the failure that produced the lane's outage in the first place. Its writer targeted a Postgres that went away, nominator_count degraded to null or to a frozen 2026-08-02 mirror covering 564 of 1,031 validators, and no probe, no red check and no exception fired. The read path degrades so gracefully that the outage was invisible. nominator_positions -- the other output of the same scan -- got a watchdog in #9273 for precisely this reason; its sibling now has the matching one, same shape as neurons/positions/chain-detail before it: one MAX() read, a pure rule, a summary rather than a throw, one exception per stale tick. An EMPTY table alerts too, since that is the state in which every nominator_count is still coming from the frozen mirror. THE POSITIONS THRESHOLD UNDERCUT ITS OWN PRODUCER. It was 6 hours, chosen while that lane had no producer at all, on the reasoning that six hours was "several missed passes at any plausible cadence". The producer now feeding it runs on a 24h tick (VALIDATOR_NOMINATORS_POLL_SECS defaults to 24*3600, and one scan writes both tables), so a healthy lane presents an age anywhere in [0h, 24h+scan] and a 6h threshold would have alerted for roughly three quarters of every day. It has not fired yet only because the table is still empty and takes the no_rows branch instead; it would have started the moment the re-enabled producer posted. An alarm that always fires is one nobody reads. Both thresholds are now 30h -- one missed pass plus slack for the scan itself (~4 min at the measured ~3,100 rows/sec) and cron jitter. Derived from the cadence, not picked, and stated as such in both headers so the next cadence change has somewhere to land. Tests pin the regression directly: a capture from the middle of a 24h cycle must be quiet. Cron 19,49 collides with nothing in workers/config.ts and stays off the */5 raw-capture and */15 probe grids; the suite asserts both that uniqueness and that wrangler.jsonc actually declares the trigger, since dispatch keys on the literal string and an undeclared cron is silently dead code. Closes #9310 Part of #9146 --- src/nominator-positions-staleness-watchdog.ts | 24 +- ...tor-nominator-counts-staleness-watchdog.ts | 142 ++++++++ ...nator-positions-staleness-watchdog.test.ts | 30 +- ...ominator-counts-staleness-watchdog.test.ts | 318 ++++++++++++++++++ workers/api.ts | 16 + workers/config.ts | 11 + wrangler.jsonc | 5 + 7 files changed, 535 insertions(+), 11 deletions(-) create mode 100644 src/validator-nominator-counts-staleness-watchdog.ts create mode 100644 tests/validator-nominator-counts-staleness-watchdog.test.ts diff --git a/src/nominator-positions-staleness-watchdog.ts b/src/nominator-positions-staleness-watchdog.ts index e070d40cf8..7d8731a59e 100644 --- a/src/nominator-positions-staleness-watchdog.ts +++ b/src/nominator-positions-staleness-watchdog.ts @@ -17,18 +17,30 @@ import { recordExceptionEvent } from "./usage-telemetry.ts"; /** * How old the ledger may get before this is a stall. * - * SIX HOURS, not the neurons lane's 45 minutes, and the difference is the + * THIRTY HOURS, not the neurons lane's 45 minutes, and the difference is the * work: this lane is a full SubtensorModule::Alpha scan (~153k rows across * every coldkey on the network), which is why it never shared the 15-minute - * metagraph cadence even when it ran. Six hours is several missed passes at - * any plausible cadence the Container runs it on, and still catches the - * failure class this exists for -- a writer that stopped entirely -- inside - * one working morning rather than never. + * metagraph cadence even when it ran. + * + * CORRECTED from six hours (#9301). Six was chosen while the lane had no + * producer at all, on the reasoning that it was "several missed passes at any + * plausible cadence" -- but the producer that now feeds it runs on a 24h tick + * (VALIDATOR_NOMINATORS_POLL_SECS defaults to 24*3600 in metagraphed-infra's + * poller, and one job's scan writes BOTH this table and + * validator_nominator_counts). A healthy lane therefore presents an age + * anywhere in [0h, 24h+scan] at any moment, so a six-hour threshold would have + * alerted for roughly three quarters of every day on a lane working perfectly + * -- the failure mode where an alarm that always fires stops being read. + * + * 30h is one missed pass plus slack for the scan itself (~4 minutes at the + * measured ~3,100 rows/sec) and cron jitter. It fires only once a pass has + * genuinely been skipped, and still catches a writer that stopped entirely + * inside a day and a quarter rather than never. * * Overridable per-deployment via NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS so * the number can follow the Container's cadence without a code deploy. */ -export const NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS = 6 * 60 * 60 * 1000; +export const NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS = 30 * 60 * 60 * 1000; export interface NominatorPositionsStalenessVerdict { stale: boolean; diff --git a/src/validator-nominator-counts-staleness-watchdog.ts b/src/validator-nominator-counts-staleness-watchdog.ts new file mode 100644 index 0000000000..385edd3d78 --- /dev/null +++ b/src/validator-nominator-counts-staleness-watchdog.ts @@ -0,0 +1,142 @@ +// The alarm for the validator-nominator-counts lane (#9301). +// +// This watchdog exists because of what happened WITHOUT one. The lane's writer +// wrote to a Postgres that was decommissioned, and nothing anywhere noticed: +// `nominator_count` on /api/v1/validators and /api/v1/validators/{hotkey} kept +// serving 200s off a frozen lakehouse mirror whose newest capture was +// 2026-08-02, covering 564 of 1,031 validators and silently losing ground as +// new ones registered. No probe, no red check, no exception -- the failure was +// invisible precisely because the read path degrades to `null` so gracefully. +// +// Same shape as src/nominator-positions-staleness-watchdog.ts deliberately -- +// its sibling from the SAME producer scan -- and as +// src/neurons-staleness-watchdog.ts before it: one MAX() read, a pure rule, a +// summary rather than a throw, and one exception event per stale tick on the +// project's alert channel. Zero alerts is the correct steady state. + +import { recordExceptionEvent } from "./usage-telemetry.ts"; + +/** + * How old the counts table may get before this is a stall. + * + * THIRTY HOURS, and the number is derived from the producer's cadence rather + * than picked: the lane is a full SubtensorModule::Alpha scan on a 24h tick + * (VALIDATOR_NOMINATORS_POLL_SECS defaults to 24*3600 in metagraphed-infra's + * poller). A healthy lane therefore presents an age anywhere in [0h, 24h+scan] + * at any moment, so any threshold at or under 24 hours alerts on a lane that + * is working perfectly. 30h is one missed pass plus slack for the scan itself + * (~4 minutes at the measured ~3,100 rows/sec) and cron jitter -- it fires + * only once a pass has genuinely been skipped, and still catches the failure + * class this exists for (a writer that stopped entirely) inside a day and a + * quarter rather than never. + * + * Overridable per-deployment via + * VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS so the number can follow + * the Container's cadence without a code deploy. + */ +export const VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS = + 30 * 60 * 60 * 1000; + +export interface ValidatorNominatorCountsStalenessVerdict { + stale: boolean; + reason: "no_rows" | "stale" | null; + age_ms: number | null; + latest_captured_at: number | null; + threshold_ms: number; +} + +/** The rule alone, testable without a database or a clock. */ +export function evaluateValidatorNominatorCountsStaleness(input: { + latestCapturedAtMs: number | null; + nowMs: number; + thresholdMs: number; +}): ValidatorNominatorCountsStalenessVerdict { + const { latestCapturedAtMs, nowMs, thresholdMs } = input; + if (latestCapturedAtMs === null) { + // An empty table is a stall of infinite age, not a healthy quiet one -- + // and here it is also the CUTOVER state, before the re-enabled lane has + // posted anything. That is exactly the condition worth alerting on: an + // empty hot tier means every nominator_count is still being filled from + // the frozen lakehouse mirror, or left null outright. + return { + stale: true, + reason: "no_rows", + age_ms: null, + latest_captured_at: null, + threshold_ms: thresholdMs, + }; + } + const age = nowMs - latestCapturedAtMs; + return { + stale: age > thresholdMs, + reason: age > thresholdMs ? "stale" : null, + age_ms: age, + latest_captured_at: latestCapturedAtMs, + threshold_ms: thresholdMs, + }; +} + +interface D1Like { + prepare(sql: string): { + first(): Promise; + }; +} + +export interface ValidatorNominatorCountsStalenessDeps { + now?: () => number; + /** Telemetry seam for tests; defaults to the real recordExceptionEvent. */ + recordException?: typeof recordExceptionEvent; +} + +/** + * One watchdog tick. Returns a summary rather than throwing, matching the + * watchdog family: a tick that cannot run is one missed report, not an outage, + * and a cron that throws is a cron nobody can read the result of. + */ +export async function runValidatorNominatorCountsStalenessWatchdog( + env: Record | null | undefined, + deps: ValidatorNominatorCountsStalenessDeps = {}, +): Promise> { + const now = deps.now ?? Date.now; + const record = deps.recordException ?? recordExceptionEvent; + const db = env?.METAGRAPH_HEALTH_DB as D1Like | undefined; + if (!db?.prepare) return { ok: false, reason: "d1 binding unavailable" }; + + const thresholdMs = + Number(env?.VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS) || + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS; + + try { + const row = (await db + .prepare( + "SELECT MAX(captured_at) AS latest FROM validator_nominator_counts", + ) + .first()) as { latest: number | null } | null; + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: row?.latest ?? null, + nowMs: now(), + thresholdMs, + }); + if (verdict.stale) { + const age = + verdict.age_ms === null + ? "no rows at all" + : `${(verdict.age_ms / 3_600_000).toFixed(1)} h old`; + await record(env as never, { + error: new Error( + `validator-nominator-counts lane stalled: latest snapshot is ${age} (threshold ${thresholdMs / 3_600_000} h) -- /validators is serving nominator_count from a table nothing is refreshing`, + ), + route: "watchdog:validator-nominator-counts-staleness", + errorCode: "stale_lane", + }).catch(() => false); + } + // `ok` describes whether the TICK ran, not whether the lane is fresh. + return { ok: true, alerted: verdict.stale, ...verdict }; + } catch (err) { + return { + ok: false, + reason: "query_failed", + detail: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/tests/nominator-positions-staleness-watchdog.test.ts b/tests/nominator-positions-staleness-watchdog.test.ts index 0ec2251977..38baa9c6be 100644 --- a/tests/nominator-positions-staleness-watchdog.test.ts +++ b/tests/nominator-positions-staleness-watchdog.test.ts @@ -48,14 +48,34 @@ describe("evaluateNominatorPositionsStaleness", () => { assert.equal(fresh.reason, null); assert.equal(fresh.age_ms, 2 * HOUR); + // 31h, not 7h: the threshold moved to 30h in #9301 once the lane had a + // real producer on a 24h tick. A 7-hour-old capture is now a HEALTHY + // mid-cycle reading -- see the constant's own header. const stalled = evaluateNominatorPositionsStaleness({ - latestCapturedAtMs: NOW - 7 * HOUR, + latestCapturedAtMs: NOW - 31 * HOUR, nowMs: NOW, thresholdMs: NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS, }); assert.equal(stalled.stale, true); assert.equal(stalled.reason, "stale"); - assert.equal(stalled.latest_captured_at, NOW - 7 * HOUR); + assert.equal(stalled.latest_captured_at, NOW - 31 * HOUR); + }); + + test("a capture from the middle of the producer's 24h cycle is quiet", () => { + // The regression #9301 fixed: at the old 6h threshold this lane alerted + // for roughly three quarters of every day while working perfectly. + for (const hours of [7, 12, 20, 23]) { + const verdict = evaluateNominatorPositionsStaleness({ + latestCapturedAtMs: NOW - hours * HOUR, + nowMs: NOW, + thresholdMs: NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS, + }); + assert.equal( + verdict.stale, + false, + `${hours}h into a 24h cycle must not alert`, + ); + } }); test("exactly at the threshold is not yet a stall", () => { @@ -120,7 +140,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { assert.equal(recorded[0]!.route, "watchdog:nominator-positions-staleness"); assert.equal(recorded[0]!.errorCode, "stale_lane"); assert.match(String(recorded[0]!.error?.message), /34\.0 h old/); - assert.match(String(recorded[0]!.error?.message), /threshold 6 h/); + assert.match(String(recorded[0]!.error?.message), /threshold 30 h/); assert.match(String(recorded[0]!.error?.message), /positions/); }); @@ -204,7 +224,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { assert.equal(empty.ok, true); assert.equal(empty.reason, "no_rows"); - const { db } = fakeDb(NOW - 12 * HOUR); + const { db } = fakeDb(NOW - 48 * HOUR); const result = await runNominatorPositionsStalenessWatchdog( { METAGRAPH_HEALTH_DB: db }, { @@ -221,7 +241,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { test("the real recordExceptionEvent default engages and no-ops unconfigured", async () => { // No telemetry env configured: the real recorder returns false without // touching the network, so the default path is exercisable in-process. - const { db } = fakeDb(NOW - 12 * HOUR); + const { db } = fakeDb(NOW - 48 * HOUR); const result = await runNominatorPositionsStalenessWatchdog( { METAGRAPH_HEALTH_DB: db }, { now: () => NOW }, diff --git a/tests/validator-nominator-counts-staleness-watchdog.test.ts b/tests/validator-nominator-counts-staleness-watchdog.test.ts new file mode 100644 index 0000000000..1c0938b960 --- /dev/null +++ b/tests/validator-nominator-counts-staleness-watchdog.test.ts @@ -0,0 +1,318 @@ +// The validator-nominator-counts lane's alarm (#9301) and its cron wiring. +// +// The rule's edges are the point, and two of them matter here. An EMPTY table +// alerts: that is the pre-cutover state, in which every nominator_count is +// still coming from a frozen lakehouse mirror or serving null outright, which +// is exactly the condition that ran unnoticed from 2026-08-02. And a capture +// from the middle of the producer's 24h cycle is QUIET -- the threshold is +// derived from that cadence rather than picked, because an alarm that fires on +// a healthy lane is one nobody reads. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, test } from "vitest"; +import { + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + evaluateValidatorNominatorCountsStaleness, + runValidatorNominatorCountsStalenessWatchdog, +} from "../src/validator-nominator-counts-staleness-watchdog.ts"; +import { handleScheduled } from "../workers/api.ts"; +import * as workerConfig from "../workers/config.ts"; + +const NOW = 1_785_800_000_000; +const HOUR = 60 * 60_000; + +function fakeDb(latest: number | null | Error) { + const queries: string[] = []; + return { + queries, + db: { + prepare(sql: string) { + queries.push(sql); + return { + async first() { + if (latest instanceof Error) throw latest; + return { latest }; + }, + }; + }, + }, + }; +} + +describe("evaluateValidatorNominatorCountsStaleness", () => { + test("a recent pass is quiet; one past the threshold is a stall", () => { + const fresh = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - 2 * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(fresh.stale, false); + assert.equal(fresh.reason, null); + assert.equal(fresh.age_ms, 2 * HOUR); + + const stalled = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - 31 * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(stalled.stale, true); + assert.equal(stalled.reason, "stale"); + assert.equal(stalled.latest_captured_at, NOW - 31 * HOUR); + }); + + test("a capture from the middle of the producer's 24h cycle is quiet", () => { + // The threshold has to clear one whole cadence: the producer scans every + // 24h, so a healthy lane presents an age anywhere in [0h, 24h+scan] and + // any threshold at or under 24h alerts on a lane that is working. + for (const hours of [7, 12, 20, 23, 24]) { + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - hours * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal( + verdict.stale, + false, + `${hours}h into a 24h cycle must not alert`, + ); + } + }); + + test("exactly at the threshold is not yet a stall", () => { + // Strictly-greater, so a lane running exactly on cadence never flaps. + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: + NOW - VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(verdict.stale, false); + }); + + test("an empty table is a stall of infinite age, never a healthy quiet", () => { + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: null, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(verdict.stale, true); + assert.equal(verdict.reason, "no_rows"); + assert.equal(verdict.age_ms, null); + assert.equal(verdict.latest_captured_at, null); + }); +}); + +describe("runValidatorNominatorCountsStalenessWatchdog", () => { + test("a fresh lane reports quiet and records nothing", async () => { + const { db, queries } = fakeDb(NOW - HOUR); + const recorded: unknown[] = []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: unknown) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, false); + assert.deepEqual(recorded, []); + assert.match( + queries[0]!, + /MAX\(captured_at\)[\s\S]*FROM validator_nominator_counts/, + ); + }); + + test("a stalled lane records ONE exception naming the age and the route it breaks", async () => { + const { db } = fakeDb(NOW - 48 * HOUR); + const recorded: { error?: Error; route?: string; errorCode?: string }[] = + []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: never) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.alerted, true); + assert.equal(recorded.length, 1); + assert.equal( + recorded[0]!.route, + "watchdog:validator-nominator-counts-staleness", + ); + assert.equal(recorded[0]!.errorCode, "stale_lane"); + assert.match(String(recorded[0]!.error?.message), /48\.0 h old/); + assert.match(String(recorded[0]!.error?.message), /threshold 30 h/); + assert.match(String(recorded[0]!.error?.message), /nominator_count/); + }); + + test("an empty table alerts with the no-rows wording", async () => { + const { db } = fakeDb(null); + const recorded: { error?: Error }[] = []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: never) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.alerted, true); + assert.equal(result.reason, "no_rows"); + assert.match(String(recorded[0]!.error?.message), /no rows at all/); + }); + + test("the env threshold override wins over the default", async () => { + const { db } = fakeDb(NOW - 2 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { + METAGRAPH_HEALTH_DB: db, + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS: String(HOUR), + }, + { now: () => NOW, recordException: (async () => true) as never }, + ); + assert.equal(result.alerted, true); + assert.equal(result.threshold_ms, HOUR); + }); + + test("a missing binding and a failing query degrade to summaries, never throw", async () => { + assert.deepEqual(await runValidatorNominatorCountsStalenessWatchdog({}), { + ok: false, + reason: "d1 binding unavailable", + }); + assert.deepEqual(await runValidatorNominatorCountsStalenessWatchdog(null), { + ok: false, + reason: "d1 binding unavailable", + }); + + const { db } = fakeDb( + new Error("D1_ERROR: no such table: validator_nominator_counts"), + ); + const failed = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { recordException: (async () => true) as never }, + ); + assert.equal(failed.ok, false); + assert.equal(failed.reason, "query_failed"); + assert.match(String(failed.detail), /no such table/); + + // A non-Error throw (D1 shims have thrown plain objects before) still + // yields a readable detail. + const stringThrow = { + prepare: () => ({ + first: async () => { + throw "socket hangup"; + }, + }), + }; + const nonError = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: stringThrow }, + { recordException: (async () => true) as never }, + ); + assert.equal(nonError.reason, "query_failed"); + assert.equal(nonError.detail, "socket hangup"); + }); + + test("a null row and a telemetry failure never fail the tick", async () => { + const nullRow = { + prepare: () => ({ first: async () => null }), + }; + const empty = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: nullRow }, + { now: () => NOW, recordException: (async () => true) as never }, + ); + assert.equal(empty.ok, true); + assert.equal(empty.reason, "no_rows"); + + const { db } = fakeDb(NOW - 48 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async () => { + throw new Error("posthog down"); + }) as never, + }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, true); + }); + + test("the real recordExceptionEvent default engages and no-ops unconfigured", async () => { + // No telemetry env configured: the real recorder returns false without + // touching the network, so the default path is exercisable in-process. + const { db } = fakeDb(NOW - 48 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { now: () => NOW }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, true); + + // The default clock is the real one, so a stamp far in the past is stale + // without injecting `now`. + const past = fakeDb(0); + const defaults = await runValidatorNominatorCountsStalenessWatchdog({ + METAGRAPH_HEALTH_DB: past.db, + }); + assert.equal(defaults.alerted, true); + }); +}); + +describe("the cron string is unique and wired", () => { + test("no other cron in workers/config.ts shares the literal string", () => { + // Dispatch keys on the LITERAL cron string, so a duplicate silently routes + // this lane into another branch entirely. + const crons = Object.entries(workerConfig) + .filter(([key]) => key.endsWith("_CRON")) + .map(([, value]) => value); + const mine = + workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON; + assert.equal( + crons.filter((cron) => cron === mine).length, + 1, + `${mine} is declared by more than one lane`, + ); + }); + + test("wrangler.jsonc declares the trigger", () => { + // A cron the Worker dispatches on but wrangler never fires is dead code -- + // and the failure is silent, since the branch simply never runs. + const raw = readFileSync( + new URL("../wrangler.jsonc", import.meta.url), + "utf8", + ) + .replace(/^\s*\/\/.*$/gm, "") + .replace(/,(\s*[}\]])/g, "$1"); + const parsed = JSON.parse(raw) as { triggers?: { crons?: string[] } }; + assert.ok( + parsed.triggers?.crons?.includes( + workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, + ), + ); + }); + + test("handleScheduled dispatches to the watchdog and returns its summary", async () => { + const { db, queries } = fakeDb(Date.now()); + const result = (await handleScheduled( + { + cron: workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, + } as unknown as ScheduledController, + { METAGRAPH_HEALTH_DB: db } as unknown as Parameters< + typeof handleScheduled + >[1], + {} as unknown as ExecutionContext, + )) as { ok: boolean; alerted: boolean }; + assert.equal(result.ok, true); + assert.equal(result.alerted, false); + assert.equal(queries.length, 1); + assert.match(queries[0]!, /FROM validator_nominator_counts/); + }); +}); diff --git a/workers/api.ts b/workers/api.ts index 35fb071854..682575fae3 100644 --- a/workers/api.ts +++ b/workers/api.ts @@ -343,6 +343,7 @@ import { checkEmissionDrift } from "../src/emission-drift-check.ts"; import { refreshLiveEconomics } from "../src/live-economics-refresh.ts"; import { runNeuronsStalenessWatchdog } from "../src/neurons-staleness-watchdog.ts"; import { runNominatorPositionsStalenessWatchdog } from "../src/nominator-positions-staleness-watchdog.ts"; +import { runValidatorNominatorCountsStalenessWatchdog } from "../src/validator-nominator-counts-staleness-watchdog.ts"; import { runChainDetailStalenessWatchdog } from "../src/chain-detail-staleness-watchdog.ts"; import { pruneChainDetail } from "../src/chain-detail-prune.ts"; import { runRpcUsageStalenessWatchdog } from "../src/rpc-usage-staleness-watchdog.ts"; @@ -409,6 +410,7 @@ import { EMISSION_DRIFT_CHECK_CRON, NEURONS_STALENESS_WATCHDOG_CRON, NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON, + VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, CHAIN_DETAIL_PRUNE_CRON, CHAIN_DETAIL_STALENESS_WATCHDOG_CRON, LIVE_ECONOMICS_REFRESH_CRON, @@ -1377,6 +1379,8 @@ function cronLabel(cron: string): string { return "neurons-staleness-watchdog"; if (cron === NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON) return "nominator-positions-staleness-watchdog"; + if (cron === VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON) + return "validator-nominator-counts-staleness-watchdog"; if (cron === CHAIN_DETAIL_PRUNE_CRON) return "chain-detail-prune"; if (cron === CHAIN_DETAIL_STALENESS_WATCHDOG_CRON) return "chain-detail-staleness-watchdog"; @@ -1655,6 +1659,18 @@ async function dispatchScheduled( env as unknown as Record, ); } + if (cron === VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON) { + // The validator-nominator-counts lane's alarm (#9301) -- the sibling of + // the watchdog above, over the other output of the same Alpha scan. Zero + // alerts is the correct steady state; a stale verdict records one + // exception under watchdog:validator-nominator-counts-staleness, the + // project's alert channel. An EMPTY table alerts too -- until the + // re-enabled lane posts, every nominator_count is still coming from the + // frozen lakehouse mirror or serving null outright. + return runValidatorNominatorCountsStalenessWatchdog( + env as unknown as Record, + ); + } if (cron === CHAIN_DETAIL_PRUNE_CRON) { // #9208 retention. Returns a summary rather than throwing so the #8998 // wrapper records the tick either way; `ok:false` on an unbound D1 or a diff --git a/workers/config.ts b/workers/config.ts index 0f7bd2da23..419118577e 100644 --- a/workers/config.ts +++ b/workers/config.ts @@ -151,6 +151,17 @@ export const NEURONS_STALENESS_WATCHDOG_CRON = "6,21,36,51 * * * *"; // the LITERAL cron string, so this must be unique here as well as matching a // wrangler.jsonc `triggers.crons` entry. export const NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON = "8,38 * * * *"; +// #9301: the validator-nominator-counts lane's alarm -- the SIBLING of the +// watchdog above, watching the other output of the same Alpha scan. It had the +// same gap for the same reason: its writer targeted a Postgres that went away, +// and `nominator_count` degraded to null (or to a frozen lakehouse mirror) +// without anything going red. Twice hourly against the same 30-hour threshold, +// since the one producer tick writes both tables. Minutes 19/49 tick on none +// of the crons in this file and stay off the */5 raw-capture and */15 probe +// grids -- dispatch keys on the LITERAL cron string, so this must be unique +// here as well as matching a wrangler.jsonc `triggers.crons` entry. +export const VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON = + "19,49 * * * *"; // #9208 retention for the chain-detail hot tier. The window only has to cover // the gap between chain tip and the decoded seam, so everything the lakehouse // has already absorbed is dropped -- see src/chain-detail-prune.ts for the diff --git a/wrangler.jsonc b/wrangler.jsonc index 4508195b91..f42994293a 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -774,6 +774,11 @@ // watchdog and lost its writer entirely without anything noticing -- see // NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON in workers/config.ts. "8,38 * * * *", + // 19,49 = the validator-nominator-counts staleness watchdog (#9301): the + // sibling of the one above, over the other output of the same Alpha + // scan, and blind for the same reason -- see + // VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON in workers/config.ts. + "19,49 * * * *", // 26 */3 = the live-economics refresh (KV economics:current), moved off // the retired refresh-economics.yml schedule -- the last Actions data // lane. Same 3-hourly cadence; :26 because :41 is taken twice over and