diff --git a/backend/src/index.ts b/backend/src/index.ts index 0e033e5..d9aac72 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -393,7 +393,9 @@ async function initializeServices() { const hsmIntegration = getHSMIntegration({ autoInitializeMasterKey: true, - enableAutoRecovery: false, + enableAutoRecovery: true, + autoRecoveryDelayMinutes: 5, + maxRecoveryAttempts: 5, auditRetentionDays: 90, }); diff --git a/backend/src/services/hsmIntegration.ts b/backend/src/services/hsmIntegration.ts index 9f7448b..d7646bf 100644 --- a/backend/src/services/hsmIntegration.ts +++ b/backend/src/services/hsmIntegration.ts @@ -10,6 +10,7 @@ export interface HSMIntegrationConfig { autoInitializeMasterKey?: boolean; enableAutoRecovery?: boolean; autoRecoveryDelayMinutes?: number; + maxRecoveryAttempts?: number; auditRetentionDays?: number; killSwitchThresholds?: { maxFailedAuth?: number; @@ -60,8 +61,9 @@ export class HSMIntegration extends EventEmitter { this.config = { autoInitializeMasterKey: true, - enableAutoRecovery: false, - autoRecoveryDelayMinutes: 30, + enableAutoRecovery: true, + autoRecoveryDelayMinutes: 5, + maxRecoveryAttempts: 5, auditRetentionDays: 90, killSwitchThresholds: { maxFailedAuth: 10, @@ -102,6 +104,7 @@ export class HSMIntegration extends EventEmitter { { autoRecoveryEnabled: this.config.enableAutoRecovery, autoRecoveryDelay: this.config.autoRecoveryDelayMinutes, + maxRecoveryAttempts: this.config.maxRecoveryAttempts, thresholds: this.config.killSwitchThresholds, }, ); @@ -372,7 +375,7 @@ export class HSMIntegration extends EventEmitter { if (updates.enableAutoRecovery !== undefined) { if (updates.enableAutoRecovery) { this.killSwitchService.enableAutoRecovery( - updates.autoRecoveryDelayMinutes || 30, + updates.autoRecoveryDelayMinutes, ); } else { this.killSwitchService.disableAutoRecovery(); diff --git a/backend/src/services/killSwitchService.ts b/backend/src/services/killSwitchService.ts index 7b7ca5a..c23d3d6 100644 --- a/backend/src/services/killSwitchService.ts +++ b/backend/src/services/killSwitchService.ts @@ -3,13 +3,14 @@ import { logger } from "../utils/logger"; import { HSMService } from "./hsmService"; import { MasterKeyManager } from "./masterKeyManager"; import { AuditService, AuditRecord } from "./auditService"; +import { killSwitchRecoveryAttempts } from "../utils/prometheus"; export interface KillSwitchTrigger { id: string; timestamp: Date; reason: string; severity: "low" | "medium" | "high" | "critical"; - source: "manual" | "automated" | "security_incident" | "system_failure"; + source: "manual" | "automated" | "security_incident" | "system_failure" | "hsm_failure"; triggeredBy?: string; metadata?: Record; } @@ -22,7 +23,9 @@ export interface KillSwitchStatus { totalActivations: number; autoRecoveryEnabled: boolean; recoveryAttempts: number; + maxRecoveryAttempts: number; nextRecoveryAttempt?: Date; + recoveredAt?: Date; } export interface SecurityMetrics { @@ -33,6 +36,15 @@ export interface SecurityMetrics { timeWindow: number; // in minutes } +// Recovery delay mapping based on trigger source (in minutes) +export const RECOVERY_DELAY_MAP: Record = { + hsm_failure: 0.5, // 30 seconds for HSM transient failure + system_failure: 2, // 2 minutes for system failures + security_incident: 5, // 5 minutes for security incidents + automated: 5, // 5 minutes for automated triggers + manual: 5, // 5 minutes for manual triggers +}; + export class KillSwitchService extends EventEmitter { private hsmService: HSMService; private masterKeyManager: MasterKeyManager; @@ -48,6 +60,8 @@ export class KillSwitchService extends EventEmitter { maxSystemErrors: number; metricsWindow: number; }; + private maxRecoveryAttempts: number; + private currentRecoveryDelay?: number; constructor( hsmService: HSMService, @@ -56,6 +70,7 @@ export class KillSwitchService extends EventEmitter { config: { autoRecoveryEnabled?: boolean; autoRecoveryDelay?: number; // minutes + maxRecoveryAttempts?: number; thresholds?: { maxFailedAuth?: number; maxSuspiciousRequests?: number; @@ -71,6 +86,8 @@ export class KillSwitchService extends EventEmitter { this.masterKeyManager = masterKeyManager; this.auditService = auditService; + this.maxRecoveryAttempts = config.maxRecoveryAttempts ?? 5; + this.thresholds = { maxFailedAuth: config.thresholds?.maxFailedAuth || 10, maxSuspiciousRequests: config.thresholds?.maxSuspiciousRequests || 5, @@ -90,12 +107,16 @@ export class KillSwitchService extends EventEmitter { this.status = { active: false, totalActivations: 0, - autoRecoveryEnabled: config.autoRecoveryEnabled ?? false, + autoRecoveryEnabled: config.autoRecoveryEnabled ?? true, recoveryAttempts: 0, + maxRecoveryAttempts: this.maxRecoveryAttempts, }; this.startMetricsCollection(); this.setupEventListeners(); + + // Initialize Prometheus gauge + killSwitchRecoveryAttempts.set(0); } private setupEventListeners(): void { @@ -103,7 +124,9 @@ export class KillSwitchService extends EventEmitter { this.hsmService.on("connectionUnhealthy", (data) => { this.recordSystemError(); if (data.error.includes("authentication")) { - this.checkThresholds("system_failure", "HSM authentication failure"); + this.checkThresholds("hsm_failure", "HSM authentication failure"); + } else { + this.checkThresholds("hsm_failure", "HSM connection unhealthy"); } }); @@ -262,16 +285,20 @@ export class KillSwitchService extends EventEmitter { this.status.lastTrigger = trigger; this.status.totalActivations++; this.status.recoveryAttempts = 0; + this.currentRecoveryDelay = undefined; // Clear master key cache this.masterKeyManager.clearCache(); - // Cancel auto-recovery if active + // Cancel auto-recovery if active and reset Prometheus gauge if (this.autoRecoveryTimer) { clearTimeout(this.autoRecoveryTimer); this.autoRecoveryTimer = null; } + // Schedule auto-recovery if enabled (start fresh for this activation) + this.scheduleAutoRecoveryIfEnabled(); + // Log the activation await this.auditService.logSecurityViolation( "kill_switch_activated", @@ -346,10 +373,27 @@ export class KillSwitchService extends EventEmitter { } } - enableAutoRecovery(delayMinutes: number = 30): void { + /** + * Enable auto-recovery with a delay based on the trigger source. + * If a delayMinutes is explicitly provided, it is used; otherwise the delay + * is derived from the last trigger's source using RECOVERY_DELAY_MAP. + */ + enableAutoRecovery(delayMinutes?: number): void { + // Store explicitly provided delay for test and backoff purposes + if (delayMinutes !== undefined) { + this.currentRecoveryDelay = delayMinutes; + } + + // Calculate delay based on trigger source if not explicitly provided + const computedDelay = + this.currentRecoveryDelay ?? + (this.status.lastTrigger + ? RECOVERY_DELAY_MAP[this.status.lastTrigger.source] ?? 5 + : 5); + this.status.autoRecoveryEnabled = true; this.status.nextRecoveryAttempt = new Date( - Date.now() + delayMinutes * 60 * 1000, + Date.now() + computedDelay * 60 * 1000, ); if (this.autoRecoveryTimer) { @@ -360,12 +404,13 @@ export class KillSwitchService extends EventEmitter { async () => { await this.attemptAutoRecovery(); }, - delayMinutes * 60 * 1000, + computedDelay * 60 * 1000, ); logger.info("Auto-recovery enabled", { - delayMinutes, + delayMinutes: computedDelay, nextAttempt: this.status.nextRecoveryAttempt, + triggerSource: this.status.lastTrigger?.source, }); } @@ -388,21 +433,104 @@ export class KillSwitchService extends EventEmitter { } this.status.recoveryAttempts++; + killSwitchRecoveryAttempts.set(this.status.recoveryAttempts); + + logger.info("Auto-recovery attempt started", { + attempt: this.status.recoveryAttempts, + maxAttempts: this.maxRecoveryAttempts, + }); + + // Check if max attempts exceeded — escalate to human operators + if (this.status.recoveryAttempts > this.maxRecoveryAttempts) { + logger.error( + "Auto-recovery: Maximum recovery attempts exceeded. Escalating to human operators.", + { + attempts: this.status.recoveryAttempts, + maxAttempts: this.maxRecoveryAttempts, + lastTrigger: this.status.lastTrigger, + }, + ); + + this.disableAutoRecovery(); + + // Log escalation to audit + await this.auditService.logSecurityViolation( + "kill_switch_recovery_escalated", + { + userId: "system", + ipAddress: "system", + userAgent: "kill-switch-service", + }, + { + type: "system", + id: `escalation-${Date.now()}`, + }, + { + reason: "Maximum auto-recovery attempts exceeded", + attempts: this.status.recoveryAttempts, + maxAttempts: this.maxRecoveryAttempts, + lastTrigger: this.status.lastTrigger, + }, + ); + + this.emit("recoveryEscalated", { + attempts: this.status.recoveryAttempts, + maxAttempts: this.maxRecoveryAttempts, + lastTrigger: this.status.lastTrigger, + }); + + return; + } try { - // Deactivate first, then verify health + // Circuit breaker pattern: Probe first — do a single health check + // while still in kill-switch state before attempting full recovery + logger.info("Auto-recovery: Running circuit-breaker probe", { + attempt: this.status.recoveryAttempts, + }); + + const probeResult = await this.runRecoveryProbe(); + + if (!probeResult.passed) { + logger.warn("Auto-recovery: Circuit-breaker probe failed", { + attempt: this.status.recoveryAttempts, + probeErrors: probeResult.errors, + }); + + await this.scheduleNextRecoveryAttempt( + `Circuit-breaker probe failed: ${probeResult.errors.join("; ")}`, + ); + return; + } + + // Probe passed — proceed with full recovery + logger.info( + "Auto-recovery: Circuit-breaker probe passed, proceeding with recovery", + { attempt: this.status.recoveryAttempts }, + ); + + // Deactivate the kill switch await this.deactivate( `Auto-recovery attempt ${this.status.recoveryAttempts}`, "system", ); + // Verify system health after deactivation const health = await this.masterKeyManager.healthCheck(); if (health.healthy) { + this.status.recoveredAt = new Date(); + killSwitchRecoveryAttempts.set(0); + logger.info("Auto-recovery successful", { attempt: this.status.recoveryAttempts, + recoveredAt: this.status.recoveredAt, + }); + + this.emit("autoRecovered", { + attempt: this.status.recoveryAttempts, + recoveredAt: this.status.recoveredAt, }); - this.emit("autoRecovered", { attempt: this.status.recoveryAttempts }); } else { logger.warn("Auto-recovery: System unhealthy after deactivation", { health, @@ -416,34 +544,124 @@ export class KillSwitchService extends EventEmitter { "high", ); - // Schedule next attempt with exponential backoff - const nextDelay = Math.min( - 30 * Math.pow(2, this.status.recoveryAttempts), - 240, - ); - this.enableAutoRecovery(nextDelay); - - this.emit("autoRecoveryFailed", { - attempt: this.status.recoveryAttempts, + await this.scheduleNextRecoveryAttempt( + `System unhealthy after deactivation`, health, - nextDelay, - }); + ); } } catch (error) { logger.error("Auto-recovery attempt failed:", error); + await this.scheduleNextRecoveryAttempt( + `Error during recovery: ${error}`, + error, + ); + } + } + + /** + * Circuit-breaker probe: runs a lightweight health check while still + * in kill-switch state to determine if it's safe to deactivate. + * Does NOT check the kill-switch status itself since it's expected to be active. + */ + private async runRecoveryProbe(): Promise<{ + passed: boolean; + errors: string[]; + }> { + const errors: string[] = []; + + try { + // Check HSM connection health (not kill-switch state) + const hsmStatus = this.hsmService.getSystemStatus(); + if (!hsmStatus.connectionHealth) { + errors.push("HSM connection unhealthy"); + } - // Schedule retry - const nextDelay = Math.min( - 30 * Math.pow(2, this.status.recoveryAttempts), - 240, + // Check master key manager health, filtering out kill-switch-related + // issues since the kill switch is expected to be active during recovery + const masterKeyHealth = await this.masterKeyManager.healthCheck(); + const relevantIssues = masterKeyHealth.issues.filter( + (issue) => !issue.toLowerCase().includes("kill switch"), ); - this.enableAutoRecovery(nextDelay); + if (relevantIssues.length > 0) { + errors.push( + `Master key manager unhealthy: ${relevantIssues.join(", ")}`, + ); + } - this.emit("autoRecoveryFailed", { - attempt: this.status.recoveryAttempts, - error, - nextDelay, + // Check if security thresholds are breached (exclude HSM kill-switch state) + if ( + this.securityMetrics.failedAuthentications >= + this.thresholds.maxFailedAuth + ) { + errors.push( + `Failed auth threshold still exceeded: ${this.securityMetrics.failedAuthentications}/${this.thresholds.maxFailedAuth}`, + ); + } + + if ( + this.securityMetrics.suspiciousRequests >= + this.thresholds.maxSuspiciousRequests + ) { + errors.push( + `Suspicious requests threshold still exceeded: ${this.securityMetrics.suspiciousRequests}/${this.thresholds.maxSuspiciousRequests}`, + ); + } + } catch (error) { + errors.push(`Probe error: ${error}`); + } + + return { + passed: errors.length === 0, + errors, + }; + } + + /** + * Schedule the next recovery attempt with exponential backoff. + * Doubles the base delay each attempt, capped at 240 minutes (4 hours). + */ + private async scheduleNextRecoveryAttempt( + reason: string, + context?: any, + ): Promise { + const baseDelay = + this.currentRecoveryDelay ?? + RECOVERY_DELAY_MAP[this.status.lastTrigger?.source ?? "automated"] ?? 5; + const nextDelay = Math.min( + baseDelay * Math.pow(2, this.status.recoveryAttempts), + 240, // cap at 4 hours + ); + + logger.warn("Auto-recovery: Scheduling next attempt with exponential backoff", { + attempt: this.status.recoveryAttempts, + nextDelay, + reason, + context, + }); + + this.enableAutoRecovery(nextDelay); + + this.emit("autoRecoveryFailed", { + attempt: this.status.recoveryAttempts, + reason, + nextDelay, + context, + }); + } + + /** + * Schedule auto-recovery on activation if autoRecoveryEnabled is true. + * Uses the trigger source to determine the initial delay. + */ + private scheduleAutoRecoveryIfEnabled(): void { + if (this.status.autoRecoveryEnabled && this.status.lastTrigger) { + const delayMinutes = + RECOVERY_DELAY_MAP[this.status.lastTrigger.source] ?? 5; + logger.info("Scheduling auto-recovery on activation", { + delayMinutes, + triggerSource: this.status.lastTrigger.source, }); + this.enableAutoRecovery(delayMinutes); } } @@ -522,6 +740,9 @@ export class KillSwitchService extends EventEmitter { clearInterval(this.metricsResetTimer); } + // Reset Prometheus gauge + killSwitchRecoveryAttempts.set(0); + logger.info("Kill switch service shutdown completed"); } } diff --git a/backend/src/tests/hsm.integration.test.ts b/backend/src/tests/hsm.integration.test.ts index b1efae5..36789bf 100644 --- a/backend/src/tests/hsm.integration.test.ts +++ b/backend/src/tests/hsm.integration.test.ts @@ -6,7 +6,8 @@ import { } from "../services/hsmService"; import { MasterKeyManager } from "../services/masterKeyManager"; import { AuditService } from "../services/auditService"; -import { KillSwitchService } from "../services/killSwitchService"; +import { KillSwitchService, RECOVERY_DELAY_MAP } from "../services/killSwitchService"; +import { killSwitchRecoveryAttempts } from "../utils/prometheus"; import { randomBytes } from "crypto"; // Mock HSM Implementation @@ -475,6 +476,161 @@ describe("HSM Integration Tests", () => { expect(status.active).toBe(false); expect(status.recoveryAttempts).toBeGreaterThan(0); }); + + test("should use circuit-breaker probe before full recovery", async () => { + await masterKeyManager.initializeMasterKey(); + + const recoveryOrder: string[] = []; + + killSwitchService.on("autoRecovered", () => { + recoveryOrder.push("autoRecovered"); + }); + + killSwitchService.on("activated", () => { + recoveryOrder.push("activated"); + }); + + await killSwitchService.activate( + "Test circuit breaker", + "manual", + "high", + ); + + // Enable auto-recovery with very short delay + killSwitchService.enableAutoRecovery(0.01); + + // Wait for auto-recovery to complete + await new Promise((resolve) => setTimeout(resolve, 1500)); + + const status = killSwitchService.getStatus(); + // With healthy HSM, recovery should succeed + expect(status.active).toBe(false); + expect(status.recoveryAttempts).toBeGreaterThan(0); + expect(recoveryOrder).toContain("autoRecovered"); + }); + + test("should escalate to human operators after max recovery attempts", async () => { + let escalated = false; + let escalatedData: any = null; + + // Create KillSwitchService with only 1 max attempt for faster testing + const fastFailKillSwitch = new KillSwitchService( + hsmService, + masterKeyManager, + auditService, + { + autoRecoveryEnabled: true, + maxRecoveryAttempts: 1, + }, + ); + + fastFailKillSwitch.on("recoveryEscalated", (data) => { + escalated = true; + escalatedData = data; + }); + + // Simulate unhealthy HSM to make recovery probe fail + const hsmStatusSpy = jest + .spyOn(hsmService, "getSystemStatus") + .mockReturnValue({ + connectionHealth: false, + killSwitchActive: true, + lastHealthCheck: new Date(), + activeKeysCount: 0, + rotationPolicy: {} as any, + }); + + await fastFailKillSwitch.activate( + "Test escalation", + "manual", + "high", + ); + + // Trigger auto-recovery with short delay + fastFailKillSwitch.enableAutoRecovery(0.01); + + // Wait for attempts to process (with exponential backoff from 0.01 base) + await new Promise((resolve) => setTimeout(resolve, 3000)); + + // Should have escalated after exceeding max attempts + expect(escalated).toBe(true); + expect(escalatedData.attempts).toBeGreaterThan(escalatedData.maxAttempts); + const status = fastFailKillSwitch.getStatus(); + expect(status.autoRecoveryEnabled).toBe(false); + + // Cleanup + hsmStatusSpy.mockRestore(); + await fastFailKillSwitch.shutdown(); + }); + + test("should use shorter recovery delay for HSM transient failure", () => { + // HSM failure should have 30 second delay + expect(RECOVERY_DELAY_MAP["hsm_failure"]).toBe(0.5); + expect(RECOVERY_DELAY_MAP["hsm_failure"] * 60).toBe(30); // 30 seconds + + // Security incident should have 5 minute delay + expect(RECOVERY_DELAY_MAP["security_incident"]).toBe(5); + expect(RECOVERY_DELAY_MAP["security_incident"] * 60).toBe(300); // 5 minutes + + // HSM failure delay should be shorter than security incident + expect(RECOVERY_DELAY_MAP["hsm_failure"]).toBeLessThan( + RECOVERY_DELAY_MAP["security_incident"], + ); + }); + + test("should use recovery delay based on trigger source", async () => { + await masterKeyManager.initializeMasterKey(); + + // Activate with HSM failure source (should get shorter delay) + await killSwitchService.activate( + "HSM transient failure", + "hsm_failure", + "high", + ); + + const statusAfterActivation = killSwitchService.getStatus(); + expect(statusAfterActivation.lastTrigger?.source).toBe("hsm_failure"); + + // Enable auto-recovery — should compute delay from source + killSwitchService.enableAutoRecovery(); + + const statusAfterRecovery = killSwitchService.getStatus(); + expect(statusAfterRecovery.autoRecoveryEnabled).toBe(true); + // Next attempt should be ~30 seconds from now (not 5 minutes) + if (statusAfterRecovery.nextRecoveryAttempt) { + const delayMs = + statusAfterRecovery.nextRecoveryAttempt.getTime() - Date.now(); + // HSM failure gets 30 second delay = 30000ms, allow small timing variance + expect(delayMs).toBeLessThan(60000); // less than 1 minute + } + + await killSwitchService.deactivate("Test cleanup"); + }); + + test("should enable auto-recovery by default", () => { + const status = killSwitchService.getStatus(); + expect(status.autoRecoveryEnabled).toBe(true); + expect(status.maxRecoveryAttempts).toBe(5); + }); + + test("should track recovery attempts in Prometheus gauge", async () => { + await masterKeyManager.initializeMasterKey(); + + // Reset gauge before test + killSwitchRecoveryAttempts.set(0); + + await killSwitchService.activate("Test prometheus gauge"); + + // Enable recovery with very short delay + killSwitchService.enableAutoRecovery(0.01); + + // Wait for recovery attempt + await new Promise((resolve) => setTimeout(resolve, 1500)); + + // After successful recovery, gauge should be reset to 0 + const gaugeValue = await killSwitchRecoveryAttempts.get(); + expect(gaugeValue.values[0]?.value).toBe(0); + }); }); describe("System Health and Monitoring", () => { diff --git a/backend/src/utils/prometheus.ts b/backend/src/utils/prometheus.ts index 5bb7859..22b313f 100644 --- a/backend/src/utils/prometheus.ts +++ b/backend/src/utils/prometheus.ts @@ -45,8 +45,8 @@ export const cacheHitRate = new promClient.Gauge({ // Create metrics for rate limiting export const rateLimitMetrics = { requestsTotal: new Counter({ - name: "http_requests_total", - help: "Total number of HTTP requests", + name: "rate_limit_requests_total", + help: "Total number of rate-limited HTTP requests", labelNames: ["method", "route", "status_code"], registers: [prometheusRegister], }), @@ -60,8 +60,8 @@ export const rateLimitMetrics = { }), activeConnections: new Gauge({ - name: "active_connections", - help: "Number of active connections", + name: "rate_limit_active_connections", + help: "Number of active connections tracked by rate limiter", registers: [prometheusRegister], }), }; @@ -98,4 +98,11 @@ export const serviceDiscoveryMetrics = { }), }; +// Kill switch specific metrics +export const killSwitchRecoveryAttempts = new Gauge({ + name: "kill_switch_recovery_attempts", + help: "Number of recovery attempts since last kill switch activation", + registers: [prometheusRegister], +}); + export default promClient;