Skip to content

perf(leaderboard): composite indexes for 30-day leaderboard queries (closes #319) #333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
whoabuddy merged 1 commit into from
Apr 8, 2026
Merged

perf(leaderboard): composite indexes for 30-day leaderboard queries (closes #319) #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions src/objects/news-do.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type { Env, Beat, Signal, SignalStatus, Streak, Brief, Classified, Classi
import { validateSlug, validateHexColor, sanitizeString, validateDateFormat } from "../lib/validators";
import { generateId, getPacificDate, getPacificYesterday, getPacificDayStartUTC, getPacificDayEndUTC, getNextDate } from "../lib/helpers";
import { CLASSIFIED_DURATION_DAYS, CLASSIFIED_BRIEF_SLOTS, CLASSIFIED_BRIEF_MAX_CHARS, CLASSIFIED_STATUSES, SIGNAL_COOLDOWN_HOURS, BEAT_EXPIRY_DAYS, MAX_SIGNALS_PER_DAY, MAX_INCLUDED_SIGNALS_PER_BRIEF, MAX_APPROVED_SIGNALS_PER_DAY, SIGNAL_STATUSES, REVIEWABLE_SIGNAL_STATUSES, CONFIG_PUBLISHER_ADDRESS, BRIEF_INCLUSION_PAYOUT_SATS, WEEKLY_PRIZE_1ST_SATS, WEEKLY_PRIZE_2ND_SATS, WEEKLY_PRIZE_3RD_SATS, SCORING_WEIGHTS, PAYMENT_STAGE_TTL_MS } from "../lib/constants";
import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL } from "./schema";
import { SCHEMA_SQL, MIGRATION_PHASE0_SQL, MIGRATION_PAYMENTS_SQL, MIGRATION_BEAT_RESTRUCTURE_SQL, MIGRATION_SBTC_TRACKING_SQL, MIGRATION_CLASSIFIEDS_CLEANUP_SQL, MIGRATION_CLASSIFIEDS_REVIEW_SQL, MIGRATION_SNAPSHOTS_SQL, MIGRATION_BEAT_CLAIMS_SQL, MIGRATION_RETRACTION_SQL, MIGRATION_BEAT_NETWORK_FOCUS_SQL, MIGRATION_BITCOIN_MACRO_SQL, MIGRATION_QUANTUM_BEAT_SQL, MIGRATION_PAYMENT_STAGING_SQL, MIGRATION_APPROVAL_CAP_INDEX_SQL, MIGRATION_BEAT_EDITORS_SQL, MIGRATION_EDITORIAL_REVIEWS_SQL, MIGRATION_EDITOR_REVIEW_RATE_SQL, MIGRATION_CURATION_CLEANUP_SQL, MIGRATION_LEADERBOARD_INDEXES_SQL } from "./schema";

// ── State machine transition maps ──
// Hoisted to module level so they are created once and are testable.
Expand Down Expand Up @@ -285,7 +285,8 @@ export class NewsDO extends DurableObject<Env> {
// 18 = Editorial reviews — type/score/factcheck columns on corrections table
// 19 = Editor review rate — editor_review_rate_sats column on beats table
// 20 = Curation cleanup — fix Mar 28-29 inscription IDs + void 312 orphaned earnings (#339)
const CURRENT_MIGRATION_VERSION = 20;
// 21 = Leaderboard composite indexes — accelerate 30-day rolling window queries (#319)
const CURRENT_MIGRATION_VERSION = 21;
const versionRows = this.ctx.storage.sql
.exec("SELECT value FROM config WHERE key = 'migration_version'")
.toArray();
Expand Down Expand Up @@ -556,6 +557,21 @@ export class NewsDO extends DurableObject<Env> {
}
}

// Leaderboard composite indexes — accelerates 30-day rolling window
// subqueries in queryLeaderboard() from 5–10 s to <500 ms (issue #319).
if (appliedVersion < 21) {
for (const stmt of MIGRATION_LEADERBOARD_INDEXES_SQL) {
try {
this.ctx.storage.sql.exec(stmt);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (!msg.includes("already exists")) {
console.error("Leaderboard indexes migration statement failed:", e);
}
}
}
}

// Record current migration version so future cold starts skip all of the above.
this.ctx.storage.sql.exec(
"INSERT INTO config (key, value) VALUES ('migration_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')",
Expand Down
21 changes: 21 additions & 0 deletions src/objects/schema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,3 +601,24 @@ export const MIGRATION_CURATION_CLEANUP_SQL = [
'db837a01-788d-4f08-a174-8758347ce61a','ffcb4de3-3998-4265-a8a2-eb8944f4af32','43800ead-c4bd-46ef-95f2-5cd1a1ae50a9','80e3529a-a4e9-4cd5-b41c-bcb701a0ba53','2b4cfe7a-75d0-4ce0-906a-bff3e09185cc','2e00e3ef-a979-4583-b010-7565d9b8e635','31cf9975-c3af-44d5-ba5c-b7e5f11375af','dc06393c-f1e0-4667-8ee0-3a593f39ff2c','248db72c-6082-43ca-8500-3c39b013c5e2','f74eb37a-cfba-47d4-adc9-06b62422e2b8','cef57500-2ee9-4c12-82eb-5cb3f8f03e52','3960c10e-92f8-43f6-b8bb-58f768dc5fc0','40d30fbb-459b-49ea-8f94-98b2c8d17a0c','52fadd57-8847-46a8-85e7-d8458f86374e','747ef5c5-30fc-4bff-a555-d52b982dcd4d','cb05bbc3-ed0c-4f77-8ec3-5c7e916bd796','772617b2-2c1b-4f61-bb50-2203b623787a','a1518f55-d566-47d0-a397-35ef9a50efd4','7d995511-db66-400c-8d26-ad198c985281','9f6d8223-aeb5-4de0-b2ab-1ecff104dcdc','545f7829-a536-464c-9417-6c06fa26d02a','bcd9e7ef-992c-4a80-b788-c7000fba15c7','14305d91-2348-4299-b26d-3a4bfebd2909','41bf9018-b15e-4995-9699-fcdb9357634f','b5e4f967-76f5-4012-9c64-c6369f010482','4f5f50e0-60f6-4901-aae7-a0468c61234b','8b866fc7-e02d-4a6f-ba88-fed94434466b','10c5c979-a1e9-45d8-92a4-59fe7d70bd2a','246983df-bd20-4d24-8dce-88b526284e82','45c3be21-17b1-41ca-ab18-82a2ec165146'
)`,
] as const;

/**
* Migration 21 — Leaderboard composite indexes.
* Adds composite indexes to accelerate the 30-day rolling window subqueries
* in queryLeaderboard(). Without these, each LEFT JOIN subquery performs a
* full table scan on every leaderboard request (10s+ for tables with 1k+ rows).
*
* Indexes added:
* - signals(correction_of, created_at) — covers the correction_of IS NULL + 30-day filter
* - brief_signals(created_at, retracted_at) — covers the 30-day window + retracted_at IS NULL filter
* - corrections(status, created_at) — covers the status='approved' + 30-day filter
* - referral_credits(credited_at) — covers the credited_at IS NOT NULL + 30-day filter
*
* Expected improvement: 5–10 s → <500 ms per arc0btc's analysis (issue #319).
*/
export const MIGRATION_LEADERBOARD_INDEXES_SQL = [
"CREATE INDEX IF NOT EXISTS idx_signals_correction_created ON signals(correction_of, created_at)",
"CREATE INDEX IF NOT EXISTS idx_brief_signals_created_retracted ON brief_signals(created_at, retracted_at)",
"CREATE INDEX IF NOT EXISTS idx_corrections_status_created ON corrections(status, created_at)",
"CREATE INDEX IF NOT EXISTS idx_referral_credits_credited ON referral_credits(credited_at)",
] as const;
Loading