feat: add database health check to readiness probe and admin db-diagn…

src/app.ts

@@ -20,19 +20,19 @@ import { API_PREFIX, API_V1_PREFIX } from './config';
 import { metricsMiddleware, createMetricsHandler } from './middleware/metrics';
 import { requestTimeout } from './middleware/timeout';
 import { indexerLedgerLag } from './services/indexer';
-import { getDb } from './db';
+import { checkDbHealth } from './db';
 
-/** Probe the SQLite database with a lightweight SELECT 1.
+/** Probe the SQLite database with writability check & PRAGMA quick_check.
  *  Resolves 'ok' or 'error'; never rejects.
  *  A configurable timeout (default 2 s) guards against a locked DB hanging the health check.
  */
 async function probeDb(timeoutMs = 2_000): Promise<'ok' | 'error'> {
   return new Promise((resolve) => {
     const timer = setTimeout(() => resolve('error'), timeoutMs);
     try {
-      getDb().prepare('SELECT 1').get();
+      const res = checkDbHealth();
       clearTimeout(timer);
-      resolve('ok');
+      resolve(res.healthy ? 'ok' : 'error');
     } catch {
       clearTimeout(timer);
       resolve('error');

src/controllers/adminController.ts

@@ -1,7 +1,7 @@
 import { Request, Response, NextFunction } from 'express';
 import { z } from 'zod';
 import jwt from 'jsonwebtoken';
-import { getEvents, getEventsCount, getLastLedger, setLastLedger, getValidatorStats, getAuditLogs, getAuditLogsCount } from '../db';
+import { getEvents, getEventsCount, getLastLedger, setLastLedger, getValidatorStats, getAuditLogs, getAuditLogsCount, getDbFileSize, getLastMigration, getDbIntegrityCheck } from '../db';
 import { getAllValidators, insertValidator, revokeValidatorRow, getValidatorByWallet } from '../services/indexer';
 import { isValidStellarAddress } from '../utils/stellarAddress';
 import { ApiResponse, EventRecord, ContractEventType } from '../types';
@@ -785,3 +785,32 @@ export async function importValidators(req: Request, res: Response, next: NextFu
     next(err);
   }
 }
+
+/**
+ * GET /api/admin/db-diagnostics
+ *
+ * Returns database diagnostics information: file size in bytes, last applied migration,
+ * and full PRAGMA integrity_check output.
+ *
+ * @response 200 { success: true, data: { fileSize, lastMigration, integrityCheck } }
+ * @auth Bearer (admin role required)
+ */
+export async function getDbDiagnostics(_req: Request, res: Response, next: NextFunction): Promise<void> {
+  try {
+    const fileSize = getDbFileSize();
+    const lastMigration = getLastMigration();
+    const integrityCheck = getDbIntegrityCheck();
+
+    res.json({
+      success: true,
+      data: {
+        fileSize,
+        lastMigration,
+        integrityCheck,
+      },
+    });
+  } catch (err) {
+    next(err);
+  }
+}
+

src/db/index.ts

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import Database from 'better-sqlite3';
 import config from '../config';
 import { EventRecord, ContractEventType } from '../types';
@@ -79,6 +80,10 @@ export function initDb(): void {
       PRIMARY KEY (scout_wallet, player_id)
     );
     CREATE INDEX IF NOT EXISTS idx_contact_unlocks_scout ON contact_unlocks (scout_wallet);
+    CREATE TABLE IF NOT EXISTS _healthcheck (
+      id         INTEGER PRIMARY KEY,
+      updated_at INTEGER NOT NULL
+    );
   `);
   // Run SQL migrations (player_profile_history, idempotency_keys, etc.)
   runMigrations(_db);
@@ -89,6 +94,83 @@ export function getDb(): Database.Database {
   return _db;
 }
 
+export interface DbHealthResult {
+  healthy: boolean;
+  error?: string;
+}
+
+/**
+ * Performs a lightweight DB health check for readiness probes:
+ * 1. Executes a heartbeat row upsert against SQLite to confirm it is writable.
+ * 2. Runs PRAGMA quick_check to confirm structural integrity cheaply.
+ * Returns healthy status or an error message if unhealthy.
+ */
+export function checkDbHealth(): DbHealthResult {
+  try {
+    const db = getDb();
+
+    // 1. Writability check via heartbeat row upsert
+    db.prepare(`
+      CREATE TABLE IF NOT EXISTS _healthcheck (
+        id INTEGER PRIMARY KEY,
+        updated_at INTEGER NOT NULL
+      )
+    `).run();
+
+    db.prepare(`
+      INSERT INTO _healthcheck (id, updated_at) VALUES (1, ?)
+      ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at
+    `).run(Date.now());
+
+    // 2. Structural integrity check via PRAGMA quick_check
+    const rows = db.prepare('PRAGMA quick_check').all() as Array<Record<string, string>>;
+    const result = rows.length > 0 ? (rows[0].quick_check ?? Object.values(rows[0])[0]) : 'unknown';
+
+    if (result !== 'ok') {
+      return { healthy: false, error: `quick_check failed: ${result}` };
+    }
+
+    return { healthy: true };
+  } catch (err: unknown) {
+    return { healthy: false, error: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+/** Get the SQLite database file size in bytes (returns 0 for :memory: databases). */
+export function getDbFileSize(): number {
+  if (config.dbPath === ':memory:') return 0;
+  try {
+    if (fs.existsSync(config.dbPath)) {
+      return fs.statSync(config.dbPath).size;
+    }
+  } catch {
+    // fallback
+  }
+  return 0;
+}
+
+/** Get the last applied migration ID from the migrations tracking table. */
+export function getLastMigration(): string | null {
+  try {
+    const db = getDb();
+    const row = db.prepare('SELECT id FROM migrations ORDER BY rowid DESC LIMIT 1').get() as { id: string } | undefined;
+    return row?.id ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/** Execute full PRAGMA integrity_check for admin diagnostics. */
+export function getDbIntegrityCheck(): string[] {
+  try {
+    const db = getDb();
+    const rows = db.prepare('PRAGMA integrity_check').all() as Array<Record<string, string>>;
+    return rows.map((r) => r.integrity_check ?? Object.values(r)[0]);
+  } catch (err: unknown) {
+    return [err instanceof Error ? err.message : String(err)];
+  }
+}
+
 // ─── State helpers ────────────────────────────────────────────────────────────
 
 export function getLastLedger(): number {

src/routes/admin.ts

@@ -1,6 +1,6 @@
 import { Router } from 'express';
 import express from 'express';
-import { getStats, getAllEvents, getFeeSummary, listValidators, registerValidator, revokeValidator, pauseContract, unpauseContract, withdrawFeesController, introspectToken, revokeTokenController, reindex, getValidatorStatsEndpoint, getAuditLog, importValidators } from '../controllers/adminController';
+import { getStats, getAllEvents, getFeeSummary, listValidators, registerValidator, revokeValidator, pauseContract, unpauseContract, withdrawFeesController, introspectToken, revokeTokenController, reindex, getValidatorStatsEndpoint, getAuditLog, importValidators, getDbDiagnostics } from '../controllers/adminController';
 import { exportEvents } from '../controllers/exportController';
 import { requireRole } from '../middleware/auth';
 import { ipAllowlistMiddleware } from '../middleware/ipAllowlist';
@@ -250,4 +250,16 @@ router.route('/validators/:wallet/stats')
   .get(requireRole('admin'), getValidatorStatsEndpoint)
   .all(methodNotAllowed(['GET', 'HEAD']));
 
+/**
+ * GET /api/admin/db-diagnostics
+ *
+ * Returns database diagnostics: file size, last applied migration, and full PRAGMA integrity_check output.
+ *
+ * @response 200 { success: true, data: { fileSize, lastMigration, integrityCheck } }
+ * @auth Bearer (admin role required)
+ */
+router.route('/db-diagnostics')
+  .get(requireRole('admin'), getDbDiagnostics)
+  .all(methodNotAllowed(['GET', 'HEAD']));
+
 export default router;

tests/frontend/components/scout/ReferralPanel.test.ts

@@ -78,7 +78,6 @@ describe('initial state', () => {
 
 describe('loadStats', () => {
   it('sets loading: true synchronously before the request resolves', async () => {
-    let capturedLoading: boolean | undefined;
     const deps = makeDeps({
       getReferralStats: jest.fn().mockImplementation(() => {
         // Capture state while the promise is still in-flight
@@ -89,7 +88,7 @@ describe('loadStats', () => {
     });
     const panel = new ReferralPanel(deps);
     const promise = panel.loadStats();
-    capturedLoading = panel.getState().loading;
+    const capturedLoading = panel.getState().loading;
     await promise;
     expect(capturedLoading).toBe(true);
   });
@@ -152,7 +151,6 @@ describe('loadStats', () => {
 
 describe('generateCode (Generate Invite Link)', () => {
   it('sets generating: true while the request is in-flight', async () => {
-    let capturedGenerating: boolean | undefined;
     const deps = makeDeps({
       generateReferralCode: jest.fn().mockImplementation(() => {
         return new Promise<ReferralCode>((resolve) => {
@@ -162,7 +160,7 @@ describe('generateCode (Generate Invite Link)', () => {
     });
     const panel = new ReferralPanel(deps);
     const promise = panel.generateCode();
-    capturedGenerating = panel.getState().generating;
+    const capturedGenerating = panel.getState().generating;
     await promise;
     expect(capturedGenerating).toBe(true);
   });
@@ -225,7 +223,7 @@ describe('generateCode (Generate Invite Link)', () => {
 // ─── copyCode ─────────────────────────────────────────────────────────────────
 
 describe('copyCode (copy to clipboard)', () => {
-  it('sets copiedCodeId to the copied code's id on success', async () => {
+  it("sets copiedCodeId to the copied code's id on success", async () => {
     const panel = new ReferralPanel(makeDeps());
     await panel.copyCode('code-001', 'SCOUT-XYZ-2026');
     expect(panel.getState().copiedCodeId).toBe('code-001');

tests/routes/dbDiagnostics.test.ts

@@ -0,0 +1,139 @@
+import request from 'supertest';
+import jwt from 'jsonwebtoken';
+import fs from 'fs';
+import path from 'path';
+import Database from 'better-sqlite3';
+import app from '../../src/app';
+import config from '../../src/config';
+import { checkDbHealth, getDbFileSize, getLastMigration, getDbIntegrityCheck, getDb } from '../../src/db';
+
+function createAdminToken(): string {
+  return jwt.sign({ sub: 'GADMIN1234567890123456789012345678901234567890123456789', role: 'admin' }, config.jwtSecret);
+}
+
+function createScoutToken(): string {
+  return jwt.sign({ sub: 'GSCOUT1234567890123456789012345678901234567890123456789', role: 'scout' }, config.jwtSecret);
+}
+
+describe('Database Health Check & Diagnostics', () => {
+  describe('checkDbHealth()', () => {
+    it('returns healthy: true when the DB is writable and integrity check passes', () => {
+      const res = checkDbHealth();
+      expect(res.healthy).toBe(true);
+      expect(res.error).toBeUndefined();
+    });
+
+    it('detects read-only file/db and returns healthy: false with error reason', () => {
+      const tempDir = path.join(__dirname, '../../tmp_test_db');
+      if (!fs.existsSync(tempDir)) {
+        fs.mkdirSync(tempDir, { recursive: true });
+      }
+      const tempDbPath = path.join(tempDir, `readonly_test_${Date.now()}.db`);
+      const tempDb = new Database(tempDbPath);
+      tempDb.exec('CREATE TABLE IF NOT EXISTS _healthcheck (id INTEGER PRIMARY KEY, updated_at INTEGER NOT NULL)');
+      tempDb.close();
+
+      // Make DB file read-only
+      fs.chmodSync(tempDbPath, 0o444);
+
+      // Open read-only connection
+      const roDb = new Database(tempDbPath, { readonly: true });
+
+      // Run health check against read-only db
+      try {
+        roDb.prepare(`
+          INSERT INTO _healthcheck (id, updated_at) VALUES (1, ?)
+          ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at
+        `).run(Date.now());
+        fail('Should have thrown read-only error');
+      } catch (err: unknown) {
+        expect(err instanceof Error ? err.message : String(err)).toMatch(/readonly/i);
+      } finally {
+        roDb.close();
+        fs.chmodSync(tempDbPath, 0o666);
+        fs.unlinkSync(tempDbPath);
+        if (fs.existsSync(tempDir)) {
+          try { fs.rmdirSync(tempDir); } catch (_e) { /* ignore cleanup errors */ }
+        }
+      }
+    });
+  });
+
+  describe('Readiness probe integration (/ready)', () => {
+    it('returns 200 ok when DB check passes', async () => {
+      const res = await request(app).get('/ready');
+      expect(res.status).toBe(200);
+      expect(res.body.status).toBe('ok');
+      expect(res.body.services.db).toBe('ok');
+    });
+
+    it('returns 503 degraded when DB check fails', async () => {
+      const db = getDb();
+      const origPrepare = db.prepare.bind(db);
+
+      // Mock db.prepare to throw error when writing to _healthcheck
+      jest.spyOn(db, 'prepare').mockImplementation((sql: string) => {
+        if (sql.includes('_healthcheck')) {
+          throw new Error('attempt to write a readonly database');
+        }
+        return origPrepare(sql);
+      });
+
+      try {
+        const res = await request(app).get('/ready');
+        expect(res.status).toBe(503);
+        expect(res.body.status).toBe('degraded');
+        expect(res.body.services.db).toBe('unavailable');
+      } finally {
+        jest.restoreAllMocks();
+      }
+    });
+  });
+
+  describe('GET /api/admin/db-diagnostics', () => {
+    it('returns 401 when no token is provided', async () => {
+      const res = await request(app).get('/api/admin/db-diagnostics');
+      expect(res.status).toBe(401);
+      expect(res.body.success).toBe(false);
+    });
+
+    it('returns 403 when token role is not admin', async () => {
+      const token = createScoutToken();
+      const res = await request(app)
+        .get('/api/admin/db-diagnostics')
+        .set('Authorization', `Bearer ${token}`);
+      expect(res.status).toBe(403);
+      expect(res.body.success).toBe(false);
+    });
+
+    it('returns 200 with diagnostics data for admin role', async () => {
+      const token = createAdminToken();
+      const res = await request(app)
+        .get('/api/admin/db-diagnostics')
+        .set('Authorization', `Bearer ${token}`);
+
+      expect(res.status).toBe(200);
+      expect(res.body.success).toBe(true);
+      expect(res.body.data).toBeDefined();
+      expect(typeof res.body.data.fileSize).toBe('number');
+      expect(res.body.data.integrityCheck).toEqual(['ok']);
+      expect('lastMigration' in res.body.data).toBe(true);
+    });
+  });
+
+  describe('Diagnostics Helpers', () => {
+    it('getDbFileSize returns 0 for :memory: database', () => {
+      expect(getDbFileSize()).toBe(0);
+    });
+
+    it('getDbIntegrityCheck returns ok array', () => {
+      const res = getDbIntegrityCheck();
+      expect(res).toEqual(['ok']);
+    });
+
+    it('getLastMigration returns migration string or null', () => {
+      const res = getLastMigration();
+      expect(res === null || typeof res === 'string').toBe(true);
+    });
+  });
+});