diff --git a/src/nominator-positions-staleness-watchdog.ts b/src/nominator-positions-staleness-watchdog.ts index e070d40cf8..7d8731a59e 100644 --- a/src/nominator-positions-staleness-watchdog.ts +++ b/src/nominator-positions-staleness-watchdog.ts @@ -17,18 +17,30 @@ import { recordExceptionEvent } from "./usage-telemetry.ts"; /** * How old the ledger may get before this is a stall. * - * SIX HOURS, not the neurons lane's 45 minutes, and the difference is the + * THIRTY HOURS, not the neurons lane's 45 minutes, and the difference is the * work: this lane is a full SubtensorModule::Alpha scan (~153k rows across * every coldkey on the network), which is why it never shared the 15-minute - * metagraph cadence even when it ran. Six hours is several missed passes at - * any plausible cadence the Container runs it on, and still catches the - * failure class this exists for -- a writer that stopped entirely -- inside - * one working morning rather than never. + * metagraph cadence even when it ran. + * + * CORRECTED from six hours (#9301). Six was chosen while the lane had no + * producer at all, on the reasoning that it was "several missed passes at any + * plausible cadence" -- but the producer that now feeds it runs on a 24h tick + * (VALIDATOR_NOMINATORS_POLL_SECS defaults to 24*3600 in metagraphed-infra's + * poller, and one job's scan writes BOTH this table and + * validator_nominator_counts). A healthy lane therefore presents an age + * anywhere in [0h, 24h+scan] at any moment, so a six-hour threshold would have + * alerted for roughly three quarters of every day on a lane working perfectly + * -- the failure mode where an alarm that always fires stops being read. + * + * 30h is one missed pass plus slack for the scan itself (~4 minutes at the + * measured ~3,100 rows/sec) and cron jitter. It fires only once a pass has + * genuinely been skipped, and still catches a writer that stopped entirely + * inside a day and a quarter rather than never. * * Overridable per-deployment via NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS so * the number can follow the Container's cadence without a code deploy. */ -export const NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS = 6 * 60 * 60 * 1000; +export const NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS = 30 * 60 * 60 * 1000; export interface NominatorPositionsStalenessVerdict { stale: boolean; diff --git a/src/validator-nominator-counts-staleness-watchdog.ts b/src/validator-nominator-counts-staleness-watchdog.ts new file mode 100644 index 0000000000..385edd3d78 --- /dev/null +++ b/src/validator-nominator-counts-staleness-watchdog.ts @@ -0,0 +1,142 @@ +// The alarm for the validator-nominator-counts lane (#9301). +// +// This watchdog exists because of what happened WITHOUT one. The lane's writer +// wrote to a Postgres that was decommissioned, and nothing anywhere noticed: +// `nominator_count` on /api/v1/validators and /api/v1/validators/{hotkey} kept +// serving 200s off a frozen lakehouse mirror whose newest capture was +// 2026-08-02, covering 564 of 1,031 validators and silently losing ground as +// new ones registered. No probe, no red check, no exception -- the failure was +// invisible precisely because the read path degrades to `null` so gracefully. +// +// Same shape as src/nominator-positions-staleness-watchdog.ts deliberately -- +// its sibling from the SAME producer scan -- and as +// src/neurons-staleness-watchdog.ts before it: one MAX() read, a pure rule, a +// summary rather than a throw, and one exception event per stale tick on the +// project's alert channel. Zero alerts is the correct steady state. + +import { recordExceptionEvent } from "./usage-telemetry.ts"; + +/** + * How old the counts table may get before this is a stall. + * + * THIRTY HOURS, and the number is derived from the producer's cadence rather + * than picked: the lane is a full SubtensorModule::Alpha scan on a 24h tick + * (VALIDATOR_NOMINATORS_POLL_SECS defaults to 24*3600 in metagraphed-infra's + * poller). A healthy lane therefore presents an age anywhere in [0h, 24h+scan] + * at any moment, so any threshold at or under 24 hours alerts on a lane that + * is working perfectly. 30h is one missed pass plus slack for the scan itself + * (~4 minutes at the measured ~3,100 rows/sec) and cron jitter -- it fires + * only once a pass has genuinely been skipped, and still catches the failure + * class this exists for (a writer that stopped entirely) inside a day and a + * quarter rather than never. + * + * Overridable per-deployment via + * VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS so the number can follow + * the Container's cadence without a code deploy. + */ +export const VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS = + 30 * 60 * 60 * 1000; + +export interface ValidatorNominatorCountsStalenessVerdict { + stale: boolean; + reason: "no_rows" | "stale" | null; + age_ms: number | null; + latest_captured_at: number | null; + threshold_ms: number; +} + +/** The rule alone, testable without a database or a clock. */ +export function evaluateValidatorNominatorCountsStaleness(input: { + latestCapturedAtMs: number | null; + nowMs: number; + thresholdMs: number; +}): ValidatorNominatorCountsStalenessVerdict { + const { latestCapturedAtMs, nowMs, thresholdMs } = input; + if (latestCapturedAtMs === null) { + // An empty table is a stall of infinite age, not a healthy quiet one -- + // and here it is also the CUTOVER state, before the re-enabled lane has + // posted anything. That is exactly the condition worth alerting on: an + // empty hot tier means every nominator_count is still being filled from + // the frozen lakehouse mirror, or left null outright. + return { + stale: true, + reason: "no_rows", + age_ms: null, + latest_captured_at: null, + threshold_ms: thresholdMs, + }; + } + const age = nowMs - latestCapturedAtMs; + return { + stale: age > thresholdMs, + reason: age > thresholdMs ? "stale" : null, + age_ms: age, + latest_captured_at: latestCapturedAtMs, + threshold_ms: thresholdMs, + }; +} + +interface D1Like { + prepare(sql: string): { + first(): Promise; + }; +} + +export interface ValidatorNominatorCountsStalenessDeps { + now?: () => number; + /** Telemetry seam for tests; defaults to the real recordExceptionEvent. */ + recordException?: typeof recordExceptionEvent; +} + +/** + * One watchdog tick. Returns a summary rather than throwing, matching the + * watchdog family: a tick that cannot run is one missed report, not an outage, + * and a cron that throws is a cron nobody can read the result of. + */ +export async function runValidatorNominatorCountsStalenessWatchdog( + env: Record | null | undefined, + deps: ValidatorNominatorCountsStalenessDeps = {}, +): Promise> { + const now = deps.now ?? Date.now; + const record = deps.recordException ?? recordExceptionEvent; + const db = env?.METAGRAPH_HEALTH_DB as D1Like | undefined; + if (!db?.prepare) return { ok: false, reason: "d1 binding unavailable" }; + + const thresholdMs = + Number(env?.VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS) || + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS; + + try { + const row = (await db + .prepare( + "SELECT MAX(captured_at) AS latest FROM validator_nominator_counts", + ) + .first()) as { latest: number | null } | null; + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: row?.latest ?? null, + nowMs: now(), + thresholdMs, + }); + if (verdict.stale) { + const age = + verdict.age_ms === null + ? "no rows at all" + : `${(verdict.age_ms / 3_600_000).toFixed(1)} h old`; + await record(env as never, { + error: new Error( + `validator-nominator-counts lane stalled: latest snapshot is ${age} (threshold ${thresholdMs / 3_600_000} h) -- /validators is serving nominator_count from a table nothing is refreshing`, + ), + route: "watchdog:validator-nominator-counts-staleness", + errorCode: "stale_lane", + }).catch(() => false); + } + // `ok` describes whether the TICK ran, not whether the lane is fresh. + return { ok: true, alerted: verdict.stale, ...verdict }; + } catch (err) { + return { + ok: false, + reason: "query_failed", + detail: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/tests/nominator-positions-staleness-watchdog.test.ts b/tests/nominator-positions-staleness-watchdog.test.ts index 0ec2251977..38baa9c6be 100644 --- a/tests/nominator-positions-staleness-watchdog.test.ts +++ b/tests/nominator-positions-staleness-watchdog.test.ts @@ -48,14 +48,34 @@ describe("evaluateNominatorPositionsStaleness", () => { assert.equal(fresh.reason, null); assert.equal(fresh.age_ms, 2 * HOUR); + // 31h, not 7h: the threshold moved to 30h in #9301 once the lane had a + // real producer on a 24h tick. A 7-hour-old capture is now a HEALTHY + // mid-cycle reading -- see the constant's own header. const stalled = evaluateNominatorPositionsStaleness({ - latestCapturedAtMs: NOW - 7 * HOUR, + latestCapturedAtMs: NOW - 31 * HOUR, nowMs: NOW, thresholdMs: NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS, }); assert.equal(stalled.stale, true); assert.equal(stalled.reason, "stale"); - assert.equal(stalled.latest_captured_at, NOW - 7 * HOUR); + assert.equal(stalled.latest_captured_at, NOW - 31 * HOUR); + }); + + test("a capture from the middle of the producer's 24h cycle is quiet", () => { + // The regression #9301 fixed: at the old 6h threshold this lane alerted + // for roughly three quarters of every day while working perfectly. + for (const hours of [7, 12, 20, 23]) { + const verdict = evaluateNominatorPositionsStaleness({ + latestCapturedAtMs: NOW - hours * HOUR, + nowMs: NOW, + thresholdMs: NOMINATOR_POSITIONS_STALENESS_THRESHOLD_MS, + }); + assert.equal( + verdict.stale, + false, + `${hours}h into a 24h cycle must not alert`, + ); + } }); test("exactly at the threshold is not yet a stall", () => { @@ -120,7 +140,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { assert.equal(recorded[0]!.route, "watchdog:nominator-positions-staleness"); assert.equal(recorded[0]!.errorCode, "stale_lane"); assert.match(String(recorded[0]!.error?.message), /34\.0 h old/); - assert.match(String(recorded[0]!.error?.message), /threshold 6 h/); + assert.match(String(recorded[0]!.error?.message), /threshold 30 h/); assert.match(String(recorded[0]!.error?.message), /positions/); }); @@ -204,7 +224,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { assert.equal(empty.ok, true); assert.equal(empty.reason, "no_rows"); - const { db } = fakeDb(NOW - 12 * HOUR); + const { db } = fakeDb(NOW - 48 * HOUR); const result = await runNominatorPositionsStalenessWatchdog( { METAGRAPH_HEALTH_DB: db }, { @@ -221,7 +241,7 @@ describe("runNominatorPositionsStalenessWatchdog", () => { test("the real recordExceptionEvent default engages and no-ops unconfigured", async () => { // No telemetry env configured: the real recorder returns false without // touching the network, so the default path is exercisable in-process. - const { db } = fakeDb(NOW - 12 * HOUR); + const { db } = fakeDb(NOW - 48 * HOUR); const result = await runNominatorPositionsStalenessWatchdog( { METAGRAPH_HEALTH_DB: db }, { now: () => NOW }, diff --git a/tests/validator-nominator-counts-staleness-watchdog.test.ts b/tests/validator-nominator-counts-staleness-watchdog.test.ts new file mode 100644 index 0000000000..1c0938b960 --- /dev/null +++ b/tests/validator-nominator-counts-staleness-watchdog.test.ts @@ -0,0 +1,318 @@ +// The validator-nominator-counts lane's alarm (#9301) and its cron wiring. +// +// The rule's edges are the point, and two of them matter here. An EMPTY table +// alerts: that is the pre-cutover state, in which every nominator_count is +// still coming from a frozen lakehouse mirror or serving null outright, which +// is exactly the condition that ran unnoticed from 2026-08-02. And a capture +// from the middle of the producer's 24h cycle is QUIET -- the threshold is +// derived from that cadence rather than picked, because an alarm that fires on +// a healthy lane is one nobody reads. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, test } from "vitest"; +import { + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + evaluateValidatorNominatorCountsStaleness, + runValidatorNominatorCountsStalenessWatchdog, +} from "../src/validator-nominator-counts-staleness-watchdog.ts"; +import { handleScheduled } from "../workers/api.ts"; +import * as workerConfig from "../workers/config.ts"; + +const NOW = 1_785_800_000_000; +const HOUR = 60 * 60_000; + +function fakeDb(latest: number | null | Error) { + const queries: string[] = []; + return { + queries, + db: { + prepare(sql: string) { + queries.push(sql); + return { + async first() { + if (latest instanceof Error) throw latest; + return { latest }; + }, + }; + }, + }, + }; +} + +describe("evaluateValidatorNominatorCountsStaleness", () => { + test("a recent pass is quiet; one past the threshold is a stall", () => { + const fresh = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - 2 * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(fresh.stale, false); + assert.equal(fresh.reason, null); + assert.equal(fresh.age_ms, 2 * HOUR); + + const stalled = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - 31 * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(stalled.stale, true); + assert.equal(stalled.reason, "stale"); + assert.equal(stalled.latest_captured_at, NOW - 31 * HOUR); + }); + + test("a capture from the middle of the producer's 24h cycle is quiet", () => { + // The threshold has to clear one whole cadence: the producer scans every + // 24h, so a healthy lane presents an age anywhere in [0h, 24h+scan] and + // any threshold at or under 24h alerts on a lane that is working. + for (const hours of [7, 12, 20, 23, 24]) { + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: NOW - hours * HOUR, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal( + verdict.stale, + false, + `${hours}h into a 24h cycle must not alert`, + ); + } + }); + + test("exactly at the threshold is not yet a stall", () => { + // Strictly-greater, so a lane running exactly on cadence never flaps. + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: + NOW - VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(verdict.stale, false); + }); + + test("an empty table is a stall of infinite age, never a healthy quiet", () => { + const verdict = evaluateValidatorNominatorCountsStaleness({ + latestCapturedAtMs: null, + nowMs: NOW, + thresholdMs: VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS, + }); + assert.equal(verdict.stale, true); + assert.equal(verdict.reason, "no_rows"); + assert.equal(verdict.age_ms, null); + assert.equal(verdict.latest_captured_at, null); + }); +}); + +describe("runValidatorNominatorCountsStalenessWatchdog", () => { + test("a fresh lane reports quiet and records nothing", async () => { + const { db, queries } = fakeDb(NOW - HOUR); + const recorded: unknown[] = []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: unknown) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, false); + assert.deepEqual(recorded, []); + assert.match( + queries[0]!, + /MAX\(captured_at\)[\s\S]*FROM validator_nominator_counts/, + ); + }); + + test("a stalled lane records ONE exception naming the age and the route it breaks", async () => { + const { db } = fakeDb(NOW - 48 * HOUR); + const recorded: { error?: Error; route?: string; errorCode?: string }[] = + []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: never) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.alerted, true); + assert.equal(recorded.length, 1); + assert.equal( + recorded[0]!.route, + "watchdog:validator-nominator-counts-staleness", + ); + assert.equal(recorded[0]!.errorCode, "stale_lane"); + assert.match(String(recorded[0]!.error?.message), /48\.0 h old/); + assert.match(String(recorded[0]!.error?.message), /threshold 30 h/); + assert.match(String(recorded[0]!.error?.message), /nominator_count/); + }); + + test("an empty table alerts with the no-rows wording", async () => { + const { db } = fakeDb(null); + const recorded: { error?: Error }[] = []; + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async (_env: never, event: never) => { + recorded.push(event); + return true; + }) as never, + }, + ); + assert.equal(result.alerted, true); + assert.equal(result.reason, "no_rows"); + assert.match(String(recorded[0]!.error?.message), /no rows at all/); + }); + + test("the env threshold override wins over the default", async () => { + const { db } = fakeDb(NOW - 2 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { + METAGRAPH_HEALTH_DB: db, + VALIDATOR_NOMINATOR_COUNTS_STALENESS_THRESHOLD_MS: String(HOUR), + }, + { now: () => NOW, recordException: (async () => true) as never }, + ); + assert.equal(result.alerted, true); + assert.equal(result.threshold_ms, HOUR); + }); + + test("a missing binding and a failing query degrade to summaries, never throw", async () => { + assert.deepEqual(await runValidatorNominatorCountsStalenessWatchdog({}), { + ok: false, + reason: "d1 binding unavailable", + }); + assert.deepEqual(await runValidatorNominatorCountsStalenessWatchdog(null), { + ok: false, + reason: "d1 binding unavailable", + }); + + const { db } = fakeDb( + new Error("D1_ERROR: no such table: validator_nominator_counts"), + ); + const failed = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { recordException: (async () => true) as never }, + ); + assert.equal(failed.ok, false); + assert.equal(failed.reason, "query_failed"); + assert.match(String(failed.detail), /no such table/); + + // A non-Error throw (D1 shims have thrown plain objects before) still + // yields a readable detail. + const stringThrow = { + prepare: () => ({ + first: async () => { + throw "socket hangup"; + }, + }), + }; + const nonError = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: stringThrow }, + { recordException: (async () => true) as never }, + ); + assert.equal(nonError.reason, "query_failed"); + assert.equal(nonError.detail, "socket hangup"); + }); + + test("a null row and a telemetry failure never fail the tick", async () => { + const nullRow = { + prepare: () => ({ first: async () => null }), + }; + const empty = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: nullRow }, + { now: () => NOW, recordException: (async () => true) as never }, + ); + assert.equal(empty.ok, true); + assert.equal(empty.reason, "no_rows"); + + const { db } = fakeDb(NOW - 48 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { + now: () => NOW, + recordException: (async () => { + throw new Error("posthog down"); + }) as never, + }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, true); + }); + + test("the real recordExceptionEvent default engages and no-ops unconfigured", async () => { + // No telemetry env configured: the real recorder returns false without + // touching the network, so the default path is exercisable in-process. + const { db } = fakeDb(NOW - 48 * HOUR); + const result = await runValidatorNominatorCountsStalenessWatchdog( + { METAGRAPH_HEALTH_DB: db }, + { now: () => NOW }, + ); + assert.equal(result.ok, true); + assert.equal(result.alerted, true); + + // The default clock is the real one, so a stamp far in the past is stale + // without injecting `now`. + const past = fakeDb(0); + const defaults = await runValidatorNominatorCountsStalenessWatchdog({ + METAGRAPH_HEALTH_DB: past.db, + }); + assert.equal(defaults.alerted, true); + }); +}); + +describe("the cron string is unique and wired", () => { + test("no other cron in workers/config.ts shares the literal string", () => { + // Dispatch keys on the LITERAL cron string, so a duplicate silently routes + // this lane into another branch entirely. + const crons = Object.entries(workerConfig) + .filter(([key]) => key.endsWith("_CRON")) + .map(([, value]) => value); + const mine = + workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON; + assert.equal( + crons.filter((cron) => cron === mine).length, + 1, + `${mine} is declared by more than one lane`, + ); + }); + + test("wrangler.jsonc declares the trigger", () => { + // A cron the Worker dispatches on but wrangler never fires is dead code -- + // and the failure is silent, since the branch simply never runs. + const raw = readFileSync( + new URL("../wrangler.jsonc", import.meta.url), + "utf8", + ) + .replace(/^\s*\/\/.*$/gm, "") + .replace(/,(\s*[}\]])/g, "$1"); + const parsed = JSON.parse(raw) as { triggers?: { crons?: string[] } }; + assert.ok( + parsed.triggers?.crons?.includes( + workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, + ), + ); + }); + + test("handleScheduled dispatches to the watchdog and returns its summary", async () => { + const { db, queries } = fakeDb(Date.now()); + const result = (await handleScheduled( + { + cron: workerConfig.VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, + } as unknown as ScheduledController, + { METAGRAPH_HEALTH_DB: db } as unknown as Parameters< + typeof handleScheduled + >[1], + {} as unknown as ExecutionContext, + )) as { ok: boolean; alerted: boolean }; + assert.equal(result.ok, true); + assert.equal(result.alerted, false); + assert.equal(queries.length, 1); + assert.match(queries[0]!, /FROM validator_nominator_counts/); + }); +}); diff --git a/workers/api.ts b/workers/api.ts index 35fb071854..682575fae3 100644 --- a/workers/api.ts +++ b/workers/api.ts @@ -343,6 +343,7 @@ import { checkEmissionDrift } from "../src/emission-drift-check.ts"; import { refreshLiveEconomics } from "../src/live-economics-refresh.ts"; import { runNeuronsStalenessWatchdog } from "../src/neurons-staleness-watchdog.ts"; import { runNominatorPositionsStalenessWatchdog } from "../src/nominator-positions-staleness-watchdog.ts"; +import { runValidatorNominatorCountsStalenessWatchdog } from "../src/validator-nominator-counts-staleness-watchdog.ts"; import { runChainDetailStalenessWatchdog } from "../src/chain-detail-staleness-watchdog.ts"; import { pruneChainDetail } from "../src/chain-detail-prune.ts"; import { runRpcUsageStalenessWatchdog } from "../src/rpc-usage-staleness-watchdog.ts"; @@ -409,6 +410,7 @@ import { EMISSION_DRIFT_CHECK_CRON, NEURONS_STALENESS_WATCHDOG_CRON, NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON, + VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON, CHAIN_DETAIL_PRUNE_CRON, CHAIN_DETAIL_STALENESS_WATCHDOG_CRON, LIVE_ECONOMICS_REFRESH_CRON, @@ -1377,6 +1379,8 @@ function cronLabel(cron: string): string { return "neurons-staleness-watchdog"; if (cron === NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON) return "nominator-positions-staleness-watchdog"; + if (cron === VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON) + return "validator-nominator-counts-staleness-watchdog"; if (cron === CHAIN_DETAIL_PRUNE_CRON) return "chain-detail-prune"; if (cron === CHAIN_DETAIL_STALENESS_WATCHDOG_CRON) return "chain-detail-staleness-watchdog"; @@ -1655,6 +1659,18 @@ async function dispatchScheduled( env as unknown as Record, ); } + if (cron === VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON) { + // The validator-nominator-counts lane's alarm (#9301) -- the sibling of + // the watchdog above, over the other output of the same Alpha scan. Zero + // alerts is the correct steady state; a stale verdict records one + // exception under watchdog:validator-nominator-counts-staleness, the + // project's alert channel. An EMPTY table alerts too -- until the + // re-enabled lane posts, every nominator_count is still coming from the + // frozen lakehouse mirror or serving null outright. + return runValidatorNominatorCountsStalenessWatchdog( + env as unknown as Record, + ); + } if (cron === CHAIN_DETAIL_PRUNE_CRON) { // #9208 retention. Returns a summary rather than throwing so the #8998 // wrapper records the tick either way; `ok:false` on an unbound D1 or a diff --git a/workers/config.ts b/workers/config.ts index 0f7bd2da23..419118577e 100644 --- a/workers/config.ts +++ b/workers/config.ts @@ -151,6 +151,17 @@ export const NEURONS_STALENESS_WATCHDOG_CRON = "6,21,36,51 * * * *"; // the LITERAL cron string, so this must be unique here as well as matching a // wrangler.jsonc `triggers.crons` entry. export const NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON = "8,38 * * * *"; +// #9301: the validator-nominator-counts lane's alarm -- the SIBLING of the +// watchdog above, watching the other output of the same Alpha scan. It had the +// same gap for the same reason: its writer targeted a Postgres that went away, +// and `nominator_count` degraded to null (or to a frozen lakehouse mirror) +// without anything going red. Twice hourly against the same 30-hour threshold, +// since the one producer tick writes both tables. Minutes 19/49 tick on none +// of the crons in this file and stay off the */5 raw-capture and */15 probe +// grids -- dispatch keys on the LITERAL cron string, so this must be unique +// here as well as matching a wrangler.jsonc `triggers.crons` entry. +export const VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON = + "19,49 * * * *"; // #9208 retention for the chain-detail hot tier. The window only has to cover // the gap between chain tip and the decoded seam, so everything the lakehouse // has already absorbed is dropped -- see src/chain-detail-prune.ts for the diff --git a/wrangler.jsonc b/wrangler.jsonc index 4508195b91..f42994293a 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -774,6 +774,11 @@ // watchdog and lost its writer entirely without anything noticing -- see // NOMINATOR_POSITIONS_STALENESS_WATCHDOG_CRON in workers/config.ts. "8,38 * * * *", + // 19,49 = the validator-nominator-counts staleness watchdog (#9301): the + // sibling of the one above, over the other output of the same Alpha + // scan, and blind for the same reason -- see + // VALIDATOR_NOMINATOR_COUNTS_STALENESS_WATCHDOG_CRON in workers/config.ts. + "19,49 * * * *", // 26 */3 = the live-economics refresh (KV economics:current), moved off // the retired refresh-economics.yml schedule -- the last Actions data // lane. Same 3-hourly cadence; :26 because :41 is taken twice over and