diff --git a/Cargo.toml b/Cargo.toml index 8b5f39e..2e015ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,10 @@ path = "tests/consensus/view_change_partition_test.rs" name = "byzantine_equivocation_recovery_test" path = "tests/consensus/byzantine_equivocation_recovery_test.rs" +[[test]] +name = "pg_pool_health_probe_test" +path = "tests/pg_pool_health_probe_test.rs" + # ── Lint policy ────────────────────────────────────────────────────────────── # Enforced via `cargo clippy -- -D warnings` in CI. The lints below are the # style/pedantic ones we deliberately do not churn on: diff --git a/src/attestation/bls_aggregator.rs b/src/attestation/bls_aggregator.rs index 1f0cac4..516c8df 100644 --- a/src/attestation/bls_aggregator.rs +++ b/src/attestation/bls_aggregator.rs @@ -279,8 +279,12 @@ impl BLSBatchVerificationCache { self.next_seq = self.next_seq.wrapping_add(1); let new_seq = self.next_seq; - let entry = - BLSCacheEntry::new(key.message_root_256, key.aggregator_index, is_valid, new_seq); + let entry = BLSCacheEntry::new( + key.message_root_256, + key.aggregator_index, + is_valid, + new_seq, + ); self.entries.insert(key, (entry, new_seq)); self.lru_order.insert((new_seq, key), ()); } @@ -321,10 +325,8 @@ pub fn truncated_prefix_32(root: &[u8; 32]) -> u32 { /// Provided for vulnerability regression and collision analysis. pub fn xor_fold_32(root: &[u8; 32]) -> u32 { let mut folded = 0u32; - for chunk in root.chunks_exact(4) { - let mut bytes = [0u8; 4]; - bytes.copy_from_slice(chunk); - folded ^= u32::from_le_bytes(bytes); + for chunk in root.as_chunks::<4>().0 { + folded ^= u32::from_le_bytes(*chunk); } folded } diff --git a/src/consensus/engine/consensus_engine.rs b/src/consensus/engine/consensus_engine.rs index d8a7314..cdbe68c 100644 --- a/src/consensus/engine/consensus_engine.rs +++ b/src/consensus/engine/consensus_engine.rs @@ -38,7 +38,7 @@ use alloc::vec::Vec; use crate::consensus::leader_election::timeout_leader::TimeoutLeader; use crate::consensus::proposal::equivocation_detector::{EquivocationDetector, EquivocationProof}; use crate::consensus::recovery::fallback_sync::{ - FallbackSyncEngine, FallbackSyncError, LockedValue, DEADLOCK_VIEW_THRESHOLD, + FallbackSyncEngine, FallbackSyncError, LockedValue, }; use crate::consensus::view_change::types::{BlockHash, PublicKey}; @@ -59,7 +59,7 @@ pub enum ConsensusEngineEvent { /// A Byzantine equivocation was detected; view was immediately advanced. EquivocationDetected { /// The equivocation proof. - proof: EquivocationProof, + proof: Box, /// View advanced to. new_view: u64, }, @@ -176,7 +176,7 @@ impl ConsensusEngine { let new_view = self.timeout_leader.current_view(); self.events .push(ConsensusEngineEvent::EquivocationDetected { - proof: proof.clone(), + proof: Box::new(proof.clone()), new_view, }); } else { @@ -278,6 +278,7 @@ impl ConsensusEngine { mod tests { use super::*; use crate::consensus::proposal::equivocation_detector::Proposal; + use crate::consensus::recovery::fallback_sync::DEADLOCK_VIEW_THRESHOLD; use crate::consensus::view_change::types::AggregateSignature; fn pk(id: u8) -> PublicKey { diff --git a/src/consensus/leader_election/timeout_leader.rs b/src/consensus/leader_election/timeout_leader.rs index 91d8b40..71286ab 100644 --- a/src/consensus/leader_election/timeout_leader.rs +++ b/src/consensus/leader_election/timeout_leader.rs @@ -128,9 +128,10 @@ impl TimeoutLeader { /// /// `timeout(v) = min(BASE_TIMEOUT_MS * 2^v, MAX_TIMEOUT_MS)` pub fn timeout_for_view(view: u64) -> u64 { - // Use saturating_mul + saturating_shl to avoid overflow on large views. + // Use saturating_mul + checked_shl to avoid overflow on large views. let shift = view.min(63); // 2^63 already overflows u64, cap the shift - let raw = BASE_TIMEOUT_MS.saturating_mul(1u64.saturating_shl(shift as u32)); + let pow2 = 1u64.checked_shl(shift as u32).unwrap_or(u64::MAX); + let raw = BASE_TIMEOUT_MS.saturating_mul(pow2); raw.min(MAX_TIMEOUT_MS) } diff --git a/src/consensus/view_change/mod.rs b/src/consensus/view_change/mod.rs index b9af7dd..e76d933 100644 --- a/src/consensus/view_change/mod.rs +++ b/src/consensus/view_change/mod.rs @@ -27,6 +27,6 @@ pub use quarantine::{QuarantineBuffer, QuarantinedQc}; pub use resolver::{create_conflict_event, QcProcessOutcome, ViewChangeResolver}; pub use types::{ compute_public_key_set_hash, AggregateSignature, BlockHash, PublicKey, QcConflictDetected, - QuorumCertificate, ViewChangeEvent, ViewChangeError, CONVERGENCE_ROUND_LIMIT, QC, + QuorumCertificate, ViewChangeError, ViewChangeEvent, CONVERGENCE_ROUND_LIMIT, QC, QUARANTINE_ROUND_LIMIT, }; diff --git a/src/consensus/view_change/resolver.rs b/src/consensus/view_change/resolver.rs index c087d14..ed42447 100644 --- a/src/consensus/view_change/resolver.rs +++ b/src/consensus/view_change/resolver.rs @@ -14,7 +14,7 @@ use core::cmp::Ordering; use crate::consensus::view_change::quarantine::QuarantineBuffer; use crate::consensus::view_change::types::{ - AggregateSignature, BlockHash, PublicKey, QcConflictDetected, ViewChangeEvent, ViewChangeError, + AggregateSignature, BlockHash, PublicKey, QcConflictDetected, ViewChangeError, ViewChangeEvent, QC, }; @@ -88,13 +88,7 @@ impl ViewChangeResolver { .checked_add(1) .ok_or(ViewChangeError::EpochOverflow)?; - let qc = QC::new( - view, - self.current_qc_epoch, - block_hash, - signers, - signature, - ); + let qc = QC::new(view, self.current_qc_epoch, block_hash, signers, signature); Ok(qc) } diff --git a/src/db/mod.rs b/src/db/mod.rs index 7bcf171..f581a22 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -5,4 +5,3 @@ pub mod committee_cache; pub mod migrations; #[path = "slashing-store.rs"] pub mod slashing_store; - diff --git a/src/db/slashing-store.rs b/src/db/slashing-store.rs index dd383fe..070eeeb 100644 --- a/src/db/slashing-store.rs +++ b/src/db/slashing-store.rs @@ -5,13 +5,13 @@ extern crate alloc; -use alloc::collections::BTreeMap; -use alloc::vec::Vec; use crate::slashing::accumulator::SlashingAccumulator; use crate::slashing::types::{ EpochIndex, GenerationalTag, SlashingRecord, ValidatorIndex, WindowOffset, DEFAULT_SLASHING_WINDOW, }; +use alloc::collections::BTreeMap; +use alloc::vec::Vec; /// Binary serialization format identifier. pub const SLASHING_STORE_MAGIC: [u8; 8] = *b"VNSLASH1"; @@ -41,7 +41,6 @@ pub struct SlashingStoreSnapshot { /// Store for persisting and querying generational slashing accumulator data. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SlashingStore { - records: BTreeMap<(ValidatorIndex, EpochIndex), SlashingRecord>, generational_tags: BTreeMap, bit_states: BTreeMap, @@ -58,7 +57,11 @@ impl SlashingStore { /// Create a new empty SlashingStore with specified window size. pub fn with_window_size(window_size: u16) -> Self { - let size = if window_size == 0 { DEFAULT_SLASHING_WINDOW as u16 } else { window_size }; + let size = if window_size == 0 { + DEFAULT_SLASHING_WINDOW as u16 + } else { + window_size + }; Self { records: BTreeMap::new(), generational_tags: BTreeMap::new(), @@ -122,7 +125,11 @@ impl SlashingStore { } /// Retrieve a slashing record by validator index and epoch. - pub fn get_record(&self, validator_index: ValidatorIndex, epoch: EpochIndex) -> Option<&SlashingRecord> { + pub fn get_record( + &self, + validator_index: ValidatorIndex, + epoch: EpochIndex, + ) -> Option<&SlashingRecord> { self.records.get(&(validator_index, epoch)) } @@ -133,7 +140,10 @@ impl SlashingStore { /// Check if a validator has bit state marked. pub fn is_slashed_bit(&self, validator_index: ValidatorIndex) -> bool { - self.bit_states.get(&validator_index).copied().unwrap_or(false) + self.bit_states + .get(&validator_index) + .copied() + .unwrap_or(false) } /// Current epoch recorded in store. @@ -215,7 +225,7 @@ impl SlashingStore { return Err(SlashingStoreError::PayloadTruncated); } - if &bytes[0..8] != &SLASHING_STORE_MAGIC { + if bytes[0..8] != SLASHING_STORE_MAGIC { return Err(SlashingStoreError::InvalidMagic); } @@ -225,12 +235,12 @@ impl SlashingStore { } let current_epoch = u64::from_be_bytes([ - bytes[12], bytes[13], bytes[14], bytes[15], - bytes[16], bytes[17], bytes[18], bytes[19], + bytes[12], bytes[13], bytes[14], bytes[15], bytes[16], bytes[17], bytes[18], bytes[19], ]); let window_generation = u16::from_be_bytes([bytes[20], bytes[21]]); let window_size = u16::from_be_bytes([bytes[22], bytes[23]]); - let record_count = u32::from_be_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]) as usize; + let record_count = + u32::from_be_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]) as usize; let record_stride = 20; // 8 + 8 + 2 + 2 let expected_len = 28 + record_count * record_stride; @@ -245,12 +255,24 @@ impl SlashingStore { let mut offset = 28; for _ in 0..record_count { let val_idx = u64::from_be_bytes([ - bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3], - bytes[offset + 4], bytes[offset + 5], bytes[offset + 6], bytes[offset + 7], + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], ]); let epoch = u64::from_be_bytes([ - bytes[offset + 8], bytes[offset + 9], bytes[offset + 10], bytes[offset + 11], - bytes[offset + 12], bytes[offset + 13], bytes[offset + 14], bytes[offset + 15], + bytes[offset + 8], + bytes[offset + 9], + bytes[offset + 10], + bytes[offset + 11], + bytes[offset + 12], + bytes[offset + 13], + bytes[offset + 14], + bytes[offset + 15], ]); let gen = u16::from_be_bytes([bytes[offset + 16], bytes[offset + 17]]); let win_offset = u16::from_be_bytes([bytes[offset + 18], bytes[offset + 19]]); diff --git a/src/lib.rs b/src/lib.rs index 48849db..8d0d715 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -125,6 +125,22 @@ pub mod cross_chain; // conservative estimate when the two models diverge by more than 10% for 3 // consecutive sync cycles. pub mod pool; + +// Memory management primitives for the shard connection-pool (issue #141). +// Buddy-system allocator that tracks contiguous free regions and coalesces +// adjacent free blocks to eliminate pathological external fragmentation under +// high-frequency tenant churn. Dependency-free and pure Rust. +pub mod mem; + +// PostgreSQL connection-pool health probe with adaptive sizing (issue #134). +// Deterministic, dependency-free primitives for probing connection-pool +// health, evaluating pool utilisation, and producing adaptive sizing +// decisions. Implements blue-green / canary deployment gates, P99 latency +// monitoring, consecutive-unhealthy-probe degradation detection, and +// system-wide dashboard snapshots. All math is pure Rust so on-chain +// contracts, off-chain monitoring agents, and deployment gates share the +// same thresholds. +pub mod pg_pool; // --- ERROR CODES --- #[contracterror] diff --git a/src/pg_pool/mod.rs b/src/pg_pool/mod.rs new file mode 100644 index 0000000..c3d60e3 --- /dev/null +++ b/src/pg_pool/mod.rs @@ -0,0 +1,1170 @@ +//! PostgreSQL connection-pool health probe with adaptive sizing (issue #134). +//! +//! This module provides deterministic, dependency-free primitives for probing +//! connection-pool health, evaluating pool capacity, and producing adaptive +//! sizing decisions. All math is pure Rust — no network I/O, no database +//! driver — so on-chain contracts, off-chain monitoring agents, and blue-green +//! deployment gates share exactly the same thresholds and logic. +//! +//! # Design overview +//! +//! ```text +//! ┌────────────────────────┐ health samples ┌──────────────────────────────┐ +//! │ ConnectionPoolState │ ────────────────▶ PoolHealthProbe │ +//! │ (per service) │ │ • probe() │ +//! │ │ │ • dashboard_snapshot() │ +//! └────────────────────────┘ └──────────────┬───────────────┘ +//! │ PoolHealthReport +//! ▼ +//! ┌──────────────────────────────┐ +//! │ PoolAdaptiveSizer │ +//! │ • recommend_resize() │ +//! │ • canary_gate() │ +//! └──────────────────────────────┘ +//! ``` +//! +//! # Operational constants +//! +//! All thresholds are tunable via [`PoolSizingConfig`] but default to the +//! values mandated by the technical bounds in issue #134: +//! * P99 critical-path latency target: < 100 ms +//! * Availability target: 99.99% (9_999 basis points) +//! * Blue-green + canary deployment gates enforced on every resize event + +extern crate alloc; + +use alloc::collections::BTreeMap; +use alloc::string::String; +use alloc::vec::Vec; + +// --------------------------------------------------------------------------- +// Operational constants +// --------------------------------------------------------------------------- + +/// P99 latency target for connection-acquisition on critical paths, in +/// milliseconds. +pub const CRITICAL_PATH_P99_MS: u64 = 100; + +/// Availability objective expressed in basis points: 99.99%. +pub const AVAILABILITY_TARGET_BPS: u32 = 9_999; + +/// Default ratio of active connections above which a pool is considered +/// saturated and a scale-out should be recommended (90%). +pub const DEFAULT_SATURATION_THRESHOLD_BPS: u32 = 9_000; + +/// Default ratio of active connections below which a pool is considered +/// under-utilised and a scale-in can be considered (30%). +pub const DEFAULT_UNDERUTILISATION_THRESHOLD_BPS: u32 = 3_000; + +/// Default minimum number of connections a pool may hold. +pub const DEFAULT_MIN_POOL_SIZE: u32 = 2; + +/// Default maximum number of connections a pool may hold. +pub const DEFAULT_MAX_POOL_SIZE: u32 = 128; + +/// Default cooldown between consecutive resize actions, in seconds. +pub const DEFAULT_RESIZE_COOLDOWN_SECS: u64 = 30; + +/// Number of consecutive unhealthy probes before a pool transitions to the +/// `Degraded` health state. +pub const DEGRADED_PROBE_THRESHOLD: u32 = 3; + +/// Number of consecutive healthy probes required to recover from `Degraded` +/// back to `Healthy`. +pub const RECOVERY_PROBE_THRESHOLD: u32 = 2; + +/// Maximum number of distinct service pools tracked concurrently. +pub const MAX_SERVICE_POOLS: usize = 512; + +/// Canary success-rate gate in basis points before a resize is promoted. +pub const CANARY_SUCCESS_TARGET_BPS: u32 = 9_999; + +// --------------------------------------------------------------------------- +// Identifier types +// --------------------------------------------------------------------------- + +/// Logical service name that owns the connection pool (e.g. `"payments"`). +pub type ServiceName = String; + +// --------------------------------------------------------------------------- +// Connection pool state snapshot +// --------------------------------------------------------------------------- + +/// A point-in-time observation of a single PostgreSQL connection pool. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConnectionPoolState { + /// Service that owns this pool. + pub service: ServiceName, + /// Maximum connections the pool is currently configured to hold. + pub pool_size: u32, + /// Connections currently checked out by application threads. + pub active_connections: u32, + /// Connections sitting idle in the pool, available for immediate use. + pub idle_connections: u32, + /// Requests currently waiting to acquire a connection (queue depth). + pub pending_requests: u32, + /// P99 connection-acquisition latency observed in the last measurement + /// window, in milliseconds. + pub p99_acquire_ms: u64, + /// Wall-clock timestamp when this snapshot was collected (seconds since + /// the Unix epoch). + pub sampled_at: u64, +} + +impl ConnectionPoolState { + /// Returns the pool utilisation in basis points (active / pool_size). + /// + /// Returns 0 when `pool_size` is zero to avoid division by zero. + pub fn utilisation_bps(&self) -> u32 { + if self.pool_size == 0 { + return 0; + } + ((self.active_connections as u64 * 10_000) / self.pool_size as u64).min(10_000) as u32 + } + + /// Returns `true` when there are requests waiting for a connection. + pub fn has_pending_requests(&self) -> bool { + self.pending_requests > 0 + } + + /// Returns total connections tracked by the pool (active + idle). + pub fn total_connections(&self) -> u32 { + self.active_connections + .saturating_add(self.idle_connections) + } +} + +// --------------------------------------------------------------------------- +// Health states +// --------------------------------------------------------------------------- + +/// Coarse health classification for a connection pool. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum PoolHealthState { + /// Pool is operating within all thresholds. + Healthy, + /// Pool is under elevated load — actionable warning, not yet degraded. + Warning, + /// Pool has been unhealthy for `DEGRADED_PROBE_THRESHOLD` consecutive + /// probes; escalation and immediate sizing action are required. + Degraded, + /// Pool is completely unavailable (pool_size == 0 or all connections + /// failed to be acquired). + Unavailable, +} + +// --------------------------------------------------------------------------- +// Resize direction +// --------------------------------------------------------------------------- + +/// Sizing recommendation produced by [`PoolAdaptiveSizer`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ResizeDecision { + /// Current pool size is appropriate — no change needed. + NoChange, + /// Increase the pool size by `delta` connections. + Expand { delta: u32 }, + /// Decrease the pool size by `delta` connections. + Shrink { delta: u32 }, +} + +// --------------------------------------------------------------------------- +// Canary analysis +// --------------------------------------------------------------------------- + +/// Canary-deployment analysis before promoting a pool resize to all pods. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PoolCanaryAnalysis { + /// Total connection-acquisition attempts recorded in the canary window. + pub acquisitions_attempted: u64, + /// Acquisitions that succeeded within the P99 latency target. + pub acquisitions_succeeded: u64, + /// Observed P99 acquisition latency in the canary window, in milliseconds. + pub p99_acquire_ms: u64, + /// Whether a security review was completed for this resize event. + pub security_review_passed: bool, +} + +impl PoolCanaryAnalysis { + /// Returns the acquisition success rate in basis points. + pub fn success_rate_bps(&self) -> u32 { + if self.acquisitions_attempted == 0 { + return 0; + } + ((self.acquisitions_succeeded.saturating_mul(10_000)) / self.acquisitions_attempted) + .min(10_000) as u32 + } + + /// Returns `Ok(())` when the canary meets all release-gate criteria. + pub fn passes_release_gate(&self) -> Result<(), PoolProbeError> { + if !self.security_review_passed { + return Err(PoolProbeError::SecurityReviewRequired); + } + if self.success_rate_bps() < CANARY_SUCCESS_TARGET_BPS + || self.p99_acquire_ms > CRITICAL_PATH_P99_MS + { + return Err(PoolProbeError::CanaryFailed); + } + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +/// Errors returned by pool-health-probe operations. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PoolProbeError { + /// No pools are registered in the registry. + NoPoolsRegistered, + /// The requested service pool was not found. + PoolNotFound, + /// Too many service pools registered concurrently. + TooManyPools, + /// A resize would violate the configured min/max pool-size bounds. + SizingBoundsViolated, + /// The resize cooldown window has not elapsed since the last action. + CooldownActive, + /// The canary analysis did not meet the release gate. + CanaryFailed, + /// A security review must be completed before the resize is promoted. + SecurityReviewRequired, +} + +// --------------------------------------------------------------------------- +// Sizing configuration +// --------------------------------------------------------------------------- + +/// Tunable parameters for the adaptive sizer. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PoolSizingConfig { + /// Utilisation (bps) above which a pool expansion is recommended. + pub saturation_threshold_bps: u32, + /// Utilisation (bps) below which a pool shrink is considered safe. + pub underutilisation_threshold_bps: u32, + /// Minimum connections the pool may hold after a shrink. + pub min_pool_size: u32, + /// Maximum connections the pool may hold after an expansion. + pub max_pool_size: u32, + /// Minimum seconds between consecutive resize actions for a pool. + pub resize_cooldown_secs: u64, +} + +impl Default for PoolSizingConfig { + fn default() -> Self { + Self { + saturation_threshold_bps: DEFAULT_SATURATION_THRESHOLD_BPS, + underutilisation_threshold_bps: DEFAULT_UNDERUTILISATION_THRESHOLD_BPS, + min_pool_size: DEFAULT_MIN_POOL_SIZE, + max_pool_size: DEFAULT_MAX_POOL_SIZE, + resize_cooldown_secs: DEFAULT_RESIZE_COOLDOWN_SECS, + } + } +} + +// --------------------------------------------------------------------------- +// Per-pool health report +// --------------------------------------------------------------------------- + +/// Result of probing a single connection pool. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PoolHealthReport { + /// The service whose pool was probed. + pub service: ServiceName, + /// Observed utilisation in basis points. + pub utilisation_bps: u32, + /// P99 connection-acquisition latency in milliseconds. + pub p99_acquire_ms: u64, + /// Number of requests waiting to acquire a connection. + pub pending_requests: u32, + /// Health state derived from the latest probe. + pub health_state: PoolHealthState, + /// Sizing recommendation for this pool. + pub resize_decision: ResizeDecision, +} + +// --------------------------------------------------------------------------- +// System-wide metrics snapshot +// --------------------------------------------------------------------------- + +/// Aggregated metrics exported to dashboards and alerting pipelines. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PoolMetricsSnapshot { + /// Number of service pools being tracked. + pub pools_tracked: usize, + /// Total active connections summed across all pools. + pub total_active_connections: u64, + /// Maximum single-pool utilisation in basis points. + pub max_pool_utilisation_bps: u32, + /// Number of pools in `Warning` or `Degraded` state. + pub unhealthy_pools: usize, + /// Number of pools in `Degraded` or `Unavailable` state. + pub critical_pools: usize, + /// P99 latency target from the operational constants. + pub p99_target_ms: u64, + /// Availability target from the operational constants. + pub availability_target_bps: u32, +} + +// --------------------------------------------------------------------------- +// Stateless health probe +// --------------------------------------------------------------------------- + +/// Stateless probe: derives health state and sizing decisions from a pool +/// snapshot and a [`PoolSizingConfig`] without retaining any state itself. +pub struct PoolHealthProbe; + +impl PoolHealthProbe { + /// Probes a single connection pool and returns a [`PoolHealthReport`]. + /// + /// # Parameters + /// * `state` — current snapshot of the pool. + /// * `config` — sizing policy to apply. + /// * `now` — current Unix timestamp in seconds. + /// * `last_resize_at` — timestamp of the last resize for this pool, or + /// `None` if none has been performed. + /// * `consecutive_unhealthy` — how many consecutive unhealthy probes have + /// been recorded for this pool (drives `Degraded` transition). + pub fn probe( + state: &ConnectionPoolState, + config: &PoolSizingConfig, + now: u64, + last_resize_at: Option, + consecutive_unhealthy: u32, + ) -> PoolHealthReport { + let utilisation_bps = state.utilisation_bps(); + let health_state = + Self::classify_health(state, config, utilisation_bps, consecutive_unhealthy); + let resize_decision = + Self::recommend_resize(state, config, now, last_resize_at, utilisation_bps); + + PoolHealthReport { + service: state.service.clone(), + utilisation_bps, + p99_acquire_ms: state.p99_acquire_ms, + pending_requests: state.pending_requests, + health_state, + resize_decision, + } + } + + /// Produces a system-wide dashboard snapshot from multiple pool states. + pub fn dashboard_snapshot( + states: &[ConnectionPoolState], + config: &PoolSizingConfig, + now: u64, + ) -> PoolMetricsSnapshot { + let pools_tracked = states.len(); + let mut total_active: u64 = 0; + let mut max_util_bps: u32 = 0; + let mut unhealthy_pools: usize = 0; + let mut critical_pools: usize = 0; + let _ = now; + + for state in states { + let util = state.utilisation_bps(); + total_active = total_active.saturating_add(state.active_connections as u64); + if util > max_util_bps { + max_util_bps = util; + } + let health = + Self::classify_health(state, config, util, 0 /* stateless snapshot */); + if health >= PoolHealthState::Warning { + unhealthy_pools += 1; + } + if health >= PoolHealthState::Degraded { + critical_pools += 1; + } + } + + PoolMetricsSnapshot { + pools_tracked, + total_active_connections: total_active, + max_pool_utilisation_bps: max_util_bps, + unhealthy_pools, + critical_pools, + p99_target_ms: CRITICAL_PATH_P99_MS, + availability_target_bps: AVAILABILITY_TARGET_BPS, + } + } + + // ----------------------------------------------------------------------- + // Private helpers + // ----------------------------------------------------------------------- + + fn classify_health( + state: &ConnectionPoolState, + config: &PoolSizingConfig, + utilisation_bps: u32, + consecutive_unhealthy: u32, + ) -> PoolHealthState { + if state.pool_size == 0 { + return PoolHealthState::Unavailable; + } + // Treat excessive P99 latency as unhealthy even when utilisation is low. + let latency_unhealthy = state.p99_acquire_ms > CRITICAL_PATH_P99_MS; + let saturated = utilisation_bps >= config.saturation_threshold_bps; + let has_pending = state.has_pending_requests(); + + if consecutive_unhealthy >= DEGRADED_PROBE_THRESHOLD { + return PoolHealthState::Degraded; + } + if saturated || has_pending || latency_unhealthy { + return PoolHealthState::Warning; + } + PoolHealthState::Healthy + } + + fn recommend_resize( + state: &ConnectionPoolState, + config: &PoolSizingConfig, + now: u64, + last_resize_at: Option, + utilisation_bps: u32, + ) -> ResizeDecision { + // Respect the cooldown window. + if let Some(last_at) = last_resize_at { + if now.saturating_sub(last_at) < config.resize_cooldown_secs { + return ResizeDecision::NoChange; + } + } + + let latency_over_target = state.p99_acquire_ms > CRITICAL_PATH_P99_MS; + + if utilisation_bps >= config.saturation_threshold_bps + || state.has_pending_requests() + || latency_over_target + { + // Expand the pool by 1 (up to the configured maximum). + if state.pool_size < config.max_pool_size { + return ResizeDecision::Expand { delta: 1 }; + } + } else if utilisation_bps < config.underutilisation_threshold_bps + && !state.has_pending_requests() + && !latency_over_target + { + // Shrink the pool by 1 (down to the configured minimum). + if state.pool_size > config.min_pool_size { + return ResizeDecision::Shrink { delta: 1 }; + } + } + + ResizeDecision::NoChange + } +} + +// --------------------------------------------------------------------------- +// Stateful adaptive sizer +// --------------------------------------------------------------------------- + +/// Stateful adaptive sizer that tracks per-pool resize history, consecutive +/// unhealthy probe counts, and enforces cooldown windows, min/max bounds, and +/// canary gates. +#[derive(Clone, Debug)] +pub struct PoolAdaptiveSizer { + config: PoolSizingConfig, + /// Maps service name → Unix timestamp of the last resize action. + last_resize_at: BTreeMap, + /// Maps service name → consecutive unhealthy probe count. + consecutive_unhealthy: BTreeMap, + /// Maps service name → consecutive healthy probe count (for recovery). + consecutive_healthy: BTreeMap, +} + +impl PoolAdaptiveSizer { + /// Creates a new adaptive sizer with the provided policy. + pub fn new(config: PoolSizingConfig) -> Self { + Self { + config, + last_resize_at: BTreeMap::new(), + consecutive_unhealthy: BTreeMap::new(), + consecutive_healthy: BTreeMap::new(), + } + } + + /// Returns the current sizing configuration. + pub fn config(&self) -> &PoolSizingConfig { + &self.config + } + + /// Probes a pool and records health / resize history. + /// + /// Returns the [`PoolHealthReport`] for the pool. When a resize action is + /// recommended, the caller is responsible for applying it; this module + /// only produces the recommendation and records the timestamp. + pub fn recommend_resize(&mut self, state: &ConnectionPoolState, now: u64) -> PoolHealthReport { + let last_at = self.last_resize_at.get(&state.service).copied(); + let consecutive = self + .consecutive_unhealthy + .get(&state.service) + .copied() + .unwrap_or(0); + + let report = PoolHealthProbe::probe(state, &self.config, now, last_at, consecutive); + + // Update consecutive unhealthy / healthy counters. + match report.health_state { + PoolHealthState::Healthy => { + // Healthy probe — increment recovery counter, reset unhealthy. + *self + .consecutive_healthy + .entry(state.service.clone()) + .or_insert(0) += 1; + let healthy_count = self + .consecutive_healthy + .get(&state.service) + .copied() + .unwrap_or(0); + if healthy_count >= RECOVERY_PROBE_THRESHOLD { + self.consecutive_unhealthy.insert(state.service.clone(), 0); + self.consecutive_healthy.insert(state.service.clone(), 0); + } + } + PoolHealthState::Warning | PoolHealthState::Degraded | PoolHealthState::Unavailable => { + *self + .consecutive_unhealthy + .entry(state.service.clone()) + .or_insert(0) += 1; + // Reset healthy counter on any unhealthy probe. + self.consecutive_healthy.insert(state.service.clone(), 0); + } + } + + // Record resize timestamp when an action is recommended. + match report.resize_decision { + ResizeDecision::Expand { .. } | ResizeDecision::Shrink { .. } => { + self.last_resize_at.insert(state.service.clone(), now); + } + ResizeDecision::NoChange => {} + } + + report + } + + /// Validates a canary gate before promoting a pool resize to all pods. + /// + /// Returns `Ok(())` when the canary meets all release criteria. + pub fn canary_gate(&self, canary: &PoolCanaryAnalysis) -> Result<(), PoolProbeError> { + canary.passes_release_gate() + } + + /// Returns the timestamp of the last resize action for `service`, or + /// `None` if none has been performed. + pub fn last_resize_at(&self, service: &str) -> Option { + self.last_resize_at.get(service).copied() + } + + /// Returns the consecutive unhealthy probe count for `service`. + pub fn consecutive_unhealthy(&self, service: &str) -> u32 { + self.consecutive_unhealthy + .get(service) + .copied() + .unwrap_or(0) + } + + /// Resets the cooldown record for a service (e.g., after a rollback). + pub fn reset_cooldown(&mut self, service: &str) { + self.last_resize_at.remove(service); + } + + /// Resets the consecutive-unhealthy counter for a service. + pub fn reset_health_counters(&mut self, service: &str) { + self.consecutive_unhealthy.remove(service); + self.consecutive_healthy.remove(service); + } +} + +impl Default for PoolAdaptiveSizer { + fn default() -> Self { + Self::new(PoolSizingConfig::default()) + } +} + +// --------------------------------------------------------------------------- +// Multi-pool registry +// --------------------------------------------------------------------------- + +/// Registry that tracks the current state and health of all monitored +/// PostgreSQL connection pools. +/// +/// This is the top-level entry point for monitoring agents: register pools, +/// update their snapshots, and query the system-wide dashboard. +#[derive(Clone, Debug)] +pub struct ConnectionPoolRegistry { + pools: BTreeMap, + sizer: PoolAdaptiveSizer, +} + +impl ConnectionPoolRegistry { + /// Creates an empty registry with the provided sizing policy. + pub fn new(config: PoolSizingConfig) -> Self { + Self { + pools: BTreeMap::new(), + sizer: PoolAdaptiveSizer::new(config), + } + } + + /// Registers or replaces a pool state snapshot. + /// + /// Returns `Err(TooManyPools)` if the registry is at capacity and the + /// service is not already registered. + pub fn upsert_pool(&mut self, state: ConnectionPoolState) -> Result<(), PoolProbeError> { + if !self.pools.contains_key(&state.service) && self.pools.len() >= MAX_SERVICE_POOLS { + return Err(PoolProbeError::TooManyPools); + } + self.pools.insert(state.service.clone(), state); + Ok(()) + } + + /// Probes all registered pools at the current timestamp and returns one + /// [`PoolHealthReport`] per pool. + pub fn probe_all(&mut self, now: u64) -> Vec { + let service_names: Vec = self.pools.keys().cloned().collect(); + let mut reports = Vec::new(); + + for name in service_names { + if let Some(state) = self.pools.get(&name) { + let state_clone = state.clone(); + let report = self.sizer.recommend_resize(&state_clone, now); + reports.push(report); + } + } + + reports + } + + /// Returns a system-wide metrics snapshot. + pub fn dashboard_snapshot(&self, now: u64) -> PoolMetricsSnapshot { + let states: Vec = self.pools.values().cloned().collect(); + PoolHealthProbe::dashboard_snapshot(&states, self.sizer.config(), now) + } + + /// Returns the current state for a specific service pool. + pub fn pool_state(&self, service: &str) -> Option<&ConnectionPoolState> { + self.pools.get(service) + } + + /// Returns a reference to the underlying adaptive sizer. + pub fn sizer(&self) -> &PoolAdaptiveSizer { + &self.sizer + } +} + +impl Default for ConnectionPoolRegistry { + fn default() -> Self { + Self::new(PoolSizingConfig::default()) + } +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + fn pool( + service: &str, + pool_size: u32, + active: u32, + idle: u32, + pending: u32, + p99_ms: u64, + ts: u64, + ) -> ConnectionPoolState { + ConnectionPoolState { + service: service.into(), + pool_size, + active_connections: active, + idle_connections: idle, + pending_requests: pending, + p99_acquire_ms: p99_ms, + sampled_at: ts, + } + } + + fn default_config() -> PoolSizingConfig { + PoolSizingConfig::default() + } + + // ----------------------------------------------------------------------- + // ConnectionPoolState helpers + // ----------------------------------------------------------------------- + + #[test] + fn utilisation_bps_is_accurate() { + let s = pool("svc", 100, 90, 10, 0, 10, 0); + assert_eq!(s.utilisation_bps(), 9_000); + } + + #[test] + fn utilisation_bps_zero_when_pool_size_is_zero() { + let s = pool("svc", 0, 0, 0, 0, 0, 0); + assert_eq!(s.utilisation_bps(), 0); + } + + #[test] + fn utilisation_bps_caps_at_ten_thousand() { + // active > pool_size (transient over-commit possible under load) + let s = pool("svc", 10, 15, 0, 0, 0, 0); + assert_eq!(s.utilisation_bps(), 10_000); + } + + #[test] + fn total_connections_is_active_plus_idle() { + let s = pool("svc", 100, 60, 30, 0, 0, 0); + assert_eq!(s.total_connections(), 90); + } + + #[test] + fn has_pending_requests_reflects_queue_depth() { + let s_idle = pool("svc", 100, 10, 80, 0, 0, 0); + let s_busy = pool("svc", 100, 100, 0, 5, 0, 0); + assert!(!s_idle.has_pending_requests()); + assert!(s_busy.has_pending_requests()); + } + + // ----------------------------------------------------------------------- + // PoolHealthProbe — health classification + // ----------------------------------------------------------------------- + + #[test] + fn healthy_when_utilisation_low_and_no_pending_requests() { + let s = pool("svc", 100, 10, 80, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Healthy); + } + + #[test] + fn warning_when_utilisation_meets_saturation_threshold() { + // 90 active / 100 pool_size = 9_000 bps == DEFAULT_SATURATION_THRESHOLD_BPS + let s = pool("svc", 100, 90, 10, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); + } + + #[test] + fn warning_when_pending_requests_nonzero() { + let s = pool("svc", 100, 50, 50, 3, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); + } + + #[test] + fn warning_when_p99_exceeds_critical_path_target() { + // p99 = 101 ms > 100 ms target + let s = pool("svc", 100, 20, 80, 0, 101, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); + } + + #[test] + fn degraded_after_consecutive_unhealthy_probe_threshold() { + let s = pool("svc", 100, 90, 10, 0, 20, 0); + let report = + PoolHealthProbe::probe(&s, &default_config(), 0, None, DEGRADED_PROBE_THRESHOLD); + assert_eq!(report.health_state, PoolHealthState::Degraded); + } + + #[test] + fn unavailable_when_pool_size_is_zero() { + let s = pool("svc", 0, 0, 0, 0, 0, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Unavailable); + } + + // ----------------------------------------------------------------------- + // PoolHealthProbe — resize decisions + // ----------------------------------------------------------------------- + + #[test] + fn expand_recommended_when_saturated() { + let s = pool("svc", 100, 90, 10, 0, 20, 0); // 9_000 bps == threshold + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + } + + #[test] + fn expand_recommended_when_pending_requests_present() { + let s = pool("svc", 100, 50, 50, 1, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + } + + #[test] + fn expand_recommended_when_latency_over_target() { + let s = pool("svc", 100, 10, 80, 0, 101, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + } + + #[test] + fn no_expand_when_pool_already_at_max_size() { + let config = PoolSizingConfig { + max_pool_size: 100, + ..default_config() + }; + let s = pool("svc", 100, 90, 10, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &config, 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); + } + + #[test] + fn shrink_recommended_when_underutilised() { + // 10 active / 100 pool_size = 1_000 bps < 3_000 bps threshold + let s = pool("svc", 100, 10, 80, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Shrink { delta: 1 }); + } + + #[test] + fn no_shrink_when_pool_already_at_min_size() { + let config = PoolSizingConfig { + min_pool_size: 100, + ..default_config() + }; + let s = pool("svc", 100, 10, 80, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &config, 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); + } + + #[test] + fn no_change_in_warning_zone_utilisation() { + // 6_000 bps — above underutilisation threshold (3_000) but below + // saturation threshold (9_000). + let s = pool("svc", 100, 60, 40, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); + } + + #[test] + fn cooldown_suppresses_resize_decision() { + let now = 1_000u64; + let last_at = Some(now - 20); // 20 s ago — within 30 s cooldown + let s = pool("svc", 100, 90, 10, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), now, last_at, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); + } + + #[test] + fn expand_allowed_after_cooldown_expires() { + let now = 1_000u64; + let last_at = Some(now - 31); // 31 s ago — cooldown elapsed + let s = pool("svc", 100, 90, 10, 0, 20, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), now, last_at, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + } + + // ----------------------------------------------------------------------- + // PoolHealthProbe — dashboard snapshot + // ----------------------------------------------------------------------- + + #[test] + fn dashboard_snapshot_aggregates_across_pools() { + let states = vec![ + pool("a", 100, 90, 10, 0, 20, 0), // saturated → Warning + pool("b", 100, 10, 80, 0, 20, 0), // low util → Healthy + pool("c", 100, 60, 40, 5, 20, 0), // pending → Warning + ]; + let snap = PoolHealthProbe::dashboard_snapshot(&states, &default_config(), 0); + + assert_eq!(snap.pools_tracked, 3); + assert_eq!(snap.total_active_connections, 160); + assert_eq!(snap.max_pool_utilisation_bps, 9_000); + assert_eq!(snap.unhealthy_pools, 2); + assert_eq!(snap.critical_pools, 0); + assert_eq!(snap.p99_target_ms, CRITICAL_PATH_P99_MS); + assert_eq!(snap.availability_target_bps, AVAILABILITY_TARGET_BPS); + } + + #[test] + fn dashboard_snapshot_with_no_pools_returns_zeroed_metrics() { + let snap = PoolHealthProbe::dashboard_snapshot(&[], &default_config(), 0); + assert_eq!(snap.pools_tracked, 0); + assert_eq!(snap.total_active_connections, 0); + assert_eq!(snap.unhealthy_pools, 0); + assert_eq!(snap.critical_pools, 0); + } + + #[test] + fn dashboard_snapshot_counts_degraded_as_critical() { + // Pool with pool_size == 0 → Unavailable (>= Degraded → critical) + let states = vec![pool("unavail", 0, 0, 0, 0, 0, 0)]; + let snap = PoolHealthProbe::dashboard_snapshot(&states, &default_config(), 0); + assert_eq!(snap.critical_pools, 1); + } + + // ----------------------------------------------------------------------- + // PoolCanaryAnalysis + // ----------------------------------------------------------------------- + + #[test] + fn canary_success_rate_is_accurate_in_basis_points() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 9_999, + p99_acquire_ms: 80, + security_review_passed: true, + }; + assert_eq!(canary.success_rate_bps(), 9_999); + assert!(canary.passes_release_gate().is_ok()); + } + + #[test] + fn canary_fails_release_gate_without_security_review() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 10_000, + p99_acquire_ms: 50, + security_review_passed: false, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::SecurityReviewRequired) + ); + } + + #[test] + fn canary_fails_release_gate_when_latency_exceeds_p99_target() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 10_000, + p99_acquire_ms: 101, + security_review_passed: true, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::CanaryFailed) + ); + } + + #[test] + fn canary_fails_release_gate_when_success_rate_too_low() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 9_990, + p99_acquire_ms: 50, + security_review_passed: true, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::CanaryFailed) + ); + } + + #[test] + fn canary_with_no_acquisitions_yields_zero_success_rate() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 0, + acquisitions_succeeded: 0, + p99_acquire_ms: 0, + security_review_passed: true, + }; + assert_eq!(canary.success_rate_bps(), 0); + } + + // ----------------------------------------------------------------------- + // PoolAdaptiveSizer + // ----------------------------------------------------------------------- + + #[test] + fn sizer_records_timestamp_after_expand() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 1_000); + let report = sizer.recommend_resize(&s, 1_000); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + assert_eq!(sizer.last_resize_at("svc"), Some(1_000)); + } + + #[test] + fn sizer_enforces_cooldown_on_consecutive_calls() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 0); + + // First call at t=1000 → triggers expand. + let r1 = sizer.recommend_resize(&s, 1_000); + assert_eq!(r1.resize_decision, ResizeDecision::Expand { delta: 1 }); + + // Second call at t=1020 → 20 s < 30 s cooldown → suppressed. + let r2 = sizer.recommend_resize(&s, 1_020); + assert_eq!(r2.resize_decision, ResizeDecision::NoChange); + } + + #[test] + fn sizer_reset_cooldown_allows_immediate_subsequent_resize() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 0); + + sizer.recommend_resize(&s, 1_000); + sizer.reset_cooldown("svc"); + + let r = sizer.recommend_resize(&s, 1_001); + assert_eq!(r.resize_decision, ResizeDecision::Expand { delta: 1 }); + } + + #[test] + fn sizer_canary_gate_accepts_valid_analysis() { + let sizer = PoolAdaptiveSizer::default(); + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 5_000, + acquisitions_succeeded: 5_000, + p99_acquire_ms: 90, + security_review_passed: true, + }; + assert!(sizer.canary_gate(&canary).is_ok()); + } + + #[test] + fn sizer_canary_gate_rejects_low_success_rate() { + let sizer = PoolAdaptiveSizer::default(); + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 5_000, + acquisitions_succeeded: 4_000, + p99_acquire_ms: 80, + security_review_passed: true, + }; + assert_eq!( + sizer.canary_gate(&canary), + Err(PoolProbeError::CanaryFailed) + ); + } + + #[test] + fn sizer_tracks_consecutive_unhealthy_probes() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 0); // Warning level + + for i in 0..DEGRADED_PROBE_THRESHOLD { + sizer.recommend_resize(&s, i as u64 * 100); + } + + assert_eq!(sizer.consecutive_unhealthy("svc"), DEGRADED_PROBE_THRESHOLD); + } + + #[test] + fn sizer_health_report_shows_degraded_after_threshold_probes() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 0); + + let mut last_report = sizer.recommend_resize(&s, 0); + for i in 1..=DEGRADED_PROBE_THRESHOLD { + last_report = sizer.recommend_resize(&s, i as u64 * 100); + } + // After DEGRADED_PROBE_THRESHOLD unhealthy probes the sizer has + // incremented the counter to DEGRADED_PROBE_THRESHOLD; the next probe + // will see the counter and classify as Degraded. + assert!(last_report.health_state >= PoolHealthState::Warning); + } + + #[test] + fn sizer_reset_health_counters_clears_degraded_state() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool("svc", 100, 90, 10, 0, 20, 0); + + for i in 0..DEGRADED_PROBE_THRESHOLD + 1 { + sizer.recommend_resize(&s, i as u64 * 100); + } + + sizer.reset_health_counters("svc"); + assert_eq!(sizer.consecutive_unhealthy("svc"), 0); + } + + // ----------------------------------------------------------------------- + // ConnectionPoolRegistry + // ----------------------------------------------------------------------- + + #[test] + fn registry_upserts_and_probes_single_pool() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool("payments", 50, 45, 5, 0, 20, 0)) + .unwrap(); + + let reports = registry.probe_all(0); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].service, "payments"); + } + + #[test] + fn registry_upsert_replaces_existing_pool_state() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool("auth", 100, 90, 10, 0, 20, 0)) + .unwrap(); + // Replace with a healthy state. + registry + .upsert_pool(pool("auth", 100, 10, 80, 0, 20, 0)) + .unwrap(); + + let snap = registry.dashboard_snapshot(0); + assert_eq!(snap.total_active_connections, 10); + assert_eq!(snap.critical_pools, 0); + } + + #[test] + fn registry_dashboard_snapshot_reflects_all_registered_pools() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool("api", 100, 90, 10, 0, 20, 0)) + .unwrap(); + registry + .upsert_pool(pool("worker", 50, 5, 40, 0, 20, 0)) + .unwrap(); + + let snap = registry.dashboard_snapshot(0); + assert_eq!(snap.pools_tracked, 2); + } + + #[test] + fn registry_pool_state_returns_none_for_unknown_service() { + let registry = ConnectionPoolRegistry::default(); + assert!(registry.pool_state("unknown").is_none()); + } + + #[test] + fn registry_probe_all_returns_one_report_per_pool() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool("svc1", 100, 50, 50, 0, 20, 0)) + .unwrap(); + registry + .upsert_pool(pool("svc2", 100, 80, 20, 0, 20, 0)) + .unwrap(); + registry + .upsert_pool(pool("svc3", 100, 10, 90, 0, 20, 0)) + .unwrap(); + + let reports = registry.probe_all(0); + assert_eq!(reports.len(), 3); + } + + // ----------------------------------------------------------------------- + // Issue #134 operational constants + // ----------------------------------------------------------------------- + + #[test] + fn issue_134_constants_match_technical_bounds() { + assert_eq!( + CRITICAL_PATH_P99_MS, 100, + "P99 critical-path target must be 100 ms" + ); + assert_eq!( + AVAILABILITY_TARGET_BPS, 9_999, + "availability target must be 99.99%" + ); + assert_eq!(DEFAULT_SATURATION_THRESHOLD_BPS, 9_000); + assert_eq!(DEFAULT_UNDERUTILISATION_THRESHOLD_BPS, 3_000); + assert_eq!(DEFAULT_MIN_POOL_SIZE, 2); + assert_eq!(DEFAULT_MAX_POOL_SIZE, 128); + assert_eq!(DEFAULT_RESIZE_COOLDOWN_SECS, 30); + assert_eq!(DEGRADED_PROBE_THRESHOLD, 3); + assert_eq!(RECOVERY_PROBE_THRESHOLD, 2); + assert_eq!(CANARY_SUCCESS_TARGET_BPS, 9_999); + } +} diff --git a/src/pool/mod.rs b/src/pool/mod.rs index 8b65078..6797e50 100644 --- a/src/pool/mod.rs +++ b/src/pool/mod.rs @@ -1,15 +1,33 @@ -//! Connection-pool capacity planning for shard nodes (issue #139). +//! Connection-pool capacity planning and shard memory management (issues #139, #141). //! -//! The `pool` module exposes the [`capacity`] sub-module, which implements the -//! two-tier capacity planning model: a per-node [`capacity::LocalEstimator`] -//! running a non-linear model (GC-pause + NUMA corrections) and a -//! [`capacity::GlobalCoordinator`] that aggregates node snapshots using a -//! linear model with a divergence-correction factor. +//! The `pool` module exposes two sub-systems: +//! +//! * [`capacity`] — two-tier capacity planning model: a per-node +//! [`capacity::LocalEstimator`] running a non-linear model (GC-pause + NUMA +//! corrections) and a [`capacity::GlobalCoordinator`] that aggregates node +//! snapshots using a linear model with a divergence-correction factor. +//! +//! * [`shard_allocator`] / [`shard_defragmenter`] / [`tenant_registry`] — shard +//! slot lifecycle management backed by the buddy-system allocator in +//! [`crate::mem`]. Handles high-frequency tenant churn with background +//! defragmentation and slot remapping (issue #141). pub mod capacity; +pub mod shard_allocator; +pub mod shard_defragmenter; +pub mod tenant_registry; pub use capacity::{ CapacityEvent, GlobalCoordinator, LocalEstimator, LocalEstimatorSnapshot, ResourceMeasurements, DIVERGENCE_CONSECUTIVE_CYCLES, DIVERGENCE_TOLERANCE, GLOBAL_COORDINATOR_SYNC_INTERVAL_S, LOCAL_ESTIMATOR_INTERVAL_S, MAX_OVERCOMMIT_RATIO, }; + +pub use shard_allocator::{ + bulk_allocate, bulk_free, PoolFragmentationGauge, ShardAllocResult, ShardAllocator, + FRAGMENTATION_ALARM_RATIO, SHARD_SIZE_BYTES, +}; + +pub use shard_defragmenter::{DefragEvent, ShardDefragmenter, COALESCING_WINDOW_MS}; + +pub use tenant_registry::TenantRegistry; diff --git a/src/slashing/accumulator.rs b/src/slashing/accumulator.rs index c6d2cc4..ed05bbf 100644 --- a/src/slashing/accumulator.rs +++ b/src/slashing/accumulator.rs @@ -18,13 +18,12 @@ extern crate alloc; -use alloc::collections::BTreeMap; -use alloc::vec::Vec; use crate::slashing::types::{ EpochIndex, GenerationalTag, SlashingAccumulatorState, SlashingRecord, ValidatorIndex, WindowOffset, DEFAULT_SLASHING_WINDOW, }; - +use alloc::collections::BTreeMap; +use alloc::vec::Vec; /// Internal validator state tracked within the accumulator. #[derive(Clone, Debug, PartialEq, Eq)] @@ -63,7 +62,11 @@ impl SlashingAccumulator { /// Create a new SlashingAccumulator with a custom window size. pub fn with_window_size(window_size: usize) -> Self { - let actual_size = if window_size == 0 { DEFAULT_SLASHING_WINDOW } else { window_size }; + let actual_size = if window_size == 0 { + DEFAULT_SLASHING_WINDOW + } else { + window_size + }; Self { window_size: actual_size, validators: BTreeMap::new(), @@ -178,7 +181,11 @@ impl SlashingAccumulator { let expected_gen = self.compute_generation(epoch); // 1. Compare bit state: must be marked slashed - let bit = self.bit_states.get(&validator_index).copied().unwrap_or(false); + let bit = self + .bit_states + .get(&validator_index) + .copied() + .unwrap_or(false); if !bit { return false; } @@ -205,7 +212,11 @@ impl SlashingAccumulator { validator_index: ValidatorIndex, current_epoch: EpochIndex, ) -> bool { - let bit = self.bit_states.get(&validator_index).copied().unwrap_or(false); + let bit = self + .bit_states + .get(&validator_index) + .copied() + .unwrap_or(false); if !bit { return false; } @@ -247,7 +258,10 @@ impl SlashingAccumulator { /// Retrieve the raw bit state for a validator. pub fn get_bit_state(&self, validator_index: ValidatorIndex) -> bool { - self.bit_states.get(&validator_index).copied().unwrap_or(false) + self.bit_states + .get(&validator_index) + .copied() + .unwrap_or(false) } /// Clear all slashing state for a validator. diff --git a/src/slashing/condition-engine.rs b/src/slashing/condition-engine.rs index 097a201..6ff2943 100644 --- a/src/slashing/condition-engine.rs +++ b/src/slashing/condition-engine.rs @@ -5,13 +5,12 @@ extern crate alloc; -use alloc::vec::Vec; use crate::slashing::accumulator::SlashingAccumulator; use crate::slashing::types::{ EpochIndex, SlashingError, SlashingRecord, ValidatorIndex, WindowOffset, DEFAULT_SLASHING_WINDOW, }; - +use alloc::vec::Vec; /// Slashing condition engine state snapshot for persistence and migrations. #[derive(Clone, Debug, PartialEq, Eq)] @@ -91,8 +90,13 @@ impl SlashingConditionEngine { /// Check if a validator is currently slashed within the historical window. #[inline] - pub fn is_slashed_in_window(&self, validator_index: ValidatorIndex, current_epoch: EpochIndex) -> bool { - self.accumulator.is_slashed_in_window(validator_index, current_epoch) + pub fn is_slashed_in_window( + &self, + validator_index: ValidatorIndex, + current_epoch: EpochIndex, + ) -> bool { + self.accumulator + .is_slashed_in_window(validator_index, current_epoch) } /// Process and record a slashing infraction for a validator. @@ -142,7 +146,8 @@ impl SlashingConditionEngine { self.accumulator = SlashingAccumulator::with_window_size(state.window_size as usize); self.current_epoch = state.current_epoch; for record in state.active_records { - self.accumulator.record_slashing(record.validator_index, record.epoch); + self.accumulator + .record_slashing(record.validator_index, record.epoch); } self.accumulator.advance_to_epoch(state.current_epoch); } diff --git a/src/slashing/mod.rs b/src/slashing/mod.rs index 5dba17c..2ab61e7 100644 --- a/src/slashing/mod.rs +++ b/src/slashing/mod.rs @@ -6,4 +6,3 @@ pub mod evidence_verifier; pub mod mempool; pub mod penalty_calculator; pub mod types; - diff --git a/src/slashing/types.rs b/src/slashing/types.rs index ee7b513..9c51ccb 100644 --- a/src/slashing/types.rs +++ b/src/slashing/types.rs @@ -239,4 +239,3 @@ pub struct RelayedSlashingEvidence { pub length: u32, pub evidence: Vec, } - diff --git a/tests/consensus/view_change_partition_test.rs b/tests/consensus/view_change_partition_test.rs index c023b4c..27717f1 100644 --- a/tests/consensus/view_change_partition_test.rs +++ b/tests/consensus/view_change_partition_test.rs @@ -52,12 +52,7 @@ fn test_partition_divergent_qcs_heal_and_converge_by_epoch() { // Partition B proposes QC_B with epoch 2 (more recent network state in partition B). let _ = node_b - .create_proposal( - 9, - make_block_hash(99), - vec![make_pk(4)], - make_sig(99), - ) + .create_proposal(9, make_block_hash(99), vec![make_pk(4)], make_sig(99)) .expect("dummy proposal to advance epoch"); let qc_b = node_b .create_proposal( @@ -220,8 +215,12 @@ fn test_partition_multi_node_cluster_simulation() { // Partition 2 generates QC_P2 for view 50 with a higher epoch. // Advance epoch in partition 2 by creating earlier proposals. - let _ = cluster[4].create_proposal(48, make_block_hash(1), vec![make_pk(4)], make_sig(1)).unwrap(); - let _ = cluster[4].create_proposal(49, make_block_hash(2), vec![make_pk(5)], make_sig(2)).unwrap(); + let _ = cluster[4] + .create_proposal(48, make_block_hash(1), vec![make_pk(4)], make_sig(1)) + .unwrap(); + let _ = cluster[4] + .create_proposal(49, make_block_hash(2), vec![make_pk(5)], make_sig(2)) + .unwrap(); let qc_p2 = cluster[4] .create_proposal( 50, diff --git a/tests/pg_pool_health_probe_test.rs b/tests/pg_pool_health_probe_test.rs new file mode 100644 index 0000000..1c908c8 --- /dev/null +++ b/tests/pg_pool_health_probe_test.rs @@ -0,0 +1,556 @@ +//! Integration tests for the PostgreSQL connection-pool health probe with +//! adaptive sizing (issue #134). +//! +//! These tests verify: +//! * Technical invariants: P99 target, availability target, and all default +//! thresholds match the bounds stated in the issue. +//! * Health-state transitions: Healthy → Warning → Degraded → Unavailable. +//! * Adaptive sizing: expansion on saturation, shrink on under-utilisation, +//! no-change in the warning zone, and cooldown enforcement. +//! * Canary gate: success-rate gate, P99 latency gate, security-review gate. +//! * Registry: upsert, probe-all, dashboard aggregation, and capacity limits. + +use sorosusu_contracts::pg_pool::{ + ConnectionPoolRegistry, ConnectionPoolState, PoolAdaptiveSizer, PoolCanaryAnalysis, + PoolHealthProbe, PoolHealthState, PoolProbeError, PoolSizingConfig, ResizeDecision, + AVAILABILITY_TARGET_BPS, CANARY_SUCCESS_TARGET_BPS, CRITICAL_PATH_P99_MS, + DEFAULT_MAX_POOL_SIZE, DEFAULT_MIN_POOL_SIZE, DEFAULT_RESIZE_COOLDOWN_SECS, + DEFAULT_SATURATION_THRESHOLD_BPS, DEFAULT_UNDERUTILISATION_THRESHOLD_BPS, + DEGRADED_PROBE_THRESHOLD, MAX_SERVICE_POOLS, RECOVERY_PROBE_THRESHOLD, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn pool_state( + service: &str, + pool_size: u32, + active: u32, + idle: u32, + pending: u32, + p99_ms: u64, +) -> ConnectionPoolState { + ConnectionPoolState { + service: service.into(), + pool_size, + active_connections: active, + idle_connections: idle, + pending_requests: pending, + p99_acquire_ms: p99_ms, + sampled_at: 0, + } +} + +fn default_config() -> PoolSizingConfig { + PoolSizingConfig::default() +} + +// --------------------------------------------------------------------------- +// Issue #134 technical invariants +// --------------------------------------------------------------------------- + +#[test] +fn issue_134_constants_match_technical_bounds() { + assert_eq!( + CRITICAL_PATH_P99_MS, 100, + "P99 critical-path target must be 100 ms" + ); + assert_eq!( + AVAILABILITY_TARGET_BPS, 9_999, + "availability target must be 99.99%" + ); + assert_eq!(DEFAULT_SATURATION_THRESHOLD_BPS, 9_000); + assert_eq!(DEFAULT_UNDERUTILISATION_THRESHOLD_BPS, 3_000); + assert_eq!(DEFAULT_MIN_POOL_SIZE, 2); + assert_eq!(DEFAULT_MAX_POOL_SIZE, 128); + assert_eq!(DEFAULT_RESIZE_COOLDOWN_SECS, 30); + assert_eq!(DEGRADED_PROBE_THRESHOLD, 3); + assert_eq!(RECOVERY_PROBE_THRESHOLD, 2); + assert_eq!(CANARY_SUCCESS_TARGET_BPS, 9_999); +} + +// --------------------------------------------------------------------------- +// ConnectionPoolState helpers +// --------------------------------------------------------------------------- + +#[test] +fn utilisation_bps_computes_correctly() { + let s = pool_state("svc", 100, 90, 10, 0, 10); + assert_eq!(s.utilisation_bps(), 9_000); +} + +#[test] +fn utilisation_bps_zero_for_empty_pool() { + let s = pool_state("svc", 0, 0, 0, 0, 0); + assert_eq!(s.utilisation_bps(), 0); +} + +#[test] +fn utilisation_bps_caps_at_ten_thousand_when_over_committed() { + let s = pool_state("svc", 10, 20, 0, 0, 0); + assert_eq!(s.utilisation_bps(), 10_000); +} + +#[test] +fn has_pending_requests_true_when_queue_depth_nonzero() { + let s = pool_state("svc", 100, 100, 0, 1, 0); + assert!(s.has_pending_requests()); +} + +#[test] +fn has_pending_requests_false_when_queue_is_empty() { + let s = pool_state("svc", 100, 50, 50, 0, 0); + assert!(!s.has_pending_requests()); +} + +#[test] +fn total_connections_is_active_plus_idle() { + let s = pool_state("svc", 100, 60, 35, 0, 0); + assert_eq!(s.total_connections(), 95); +} + +// --------------------------------------------------------------------------- +// PoolHealthProbe — health state classification +// --------------------------------------------------------------------------- + +#[test] +fn probe_healthy_under_normal_load() { + // 50% utilisation, no pending, P99 well within target. + let s = pool_state("auth", 100, 50, 50, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Healthy); +} + +#[test] +fn probe_warning_at_saturation_threshold() { + // 9_000 bps == DEFAULT_SATURATION_THRESHOLD_BPS + let s = pool_state("auth", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); +} + +#[test] +fn probe_warning_above_saturation_threshold() { + let s = pool_state("auth", 100, 95, 5, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); +} + +#[test] +fn probe_warning_when_requests_are_queued() { + let s = pool_state("auth", 100, 40, 60, 2, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); +} + +#[test] +fn probe_warning_when_p99_latency_exceeds_100ms() { + let s = pool_state("auth", 100, 20, 80, 0, 101); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Warning); +} + +#[test] +fn probe_healthy_when_p99_exactly_at_target() { + let s = pool_state("auth", 100, 20, 80, 0, 100); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Healthy); +} + +#[test] +fn probe_degraded_after_threshold_consecutive_unhealthy_probes() { + let s = pool_state("auth", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, DEGRADED_PROBE_THRESHOLD); + assert_eq!(report.health_state, PoolHealthState::Degraded); +} + +#[test] +fn probe_unavailable_when_pool_has_no_connections_configured() { + let s = pool_state("auth", 0, 0, 0, 0, 0); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.health_state, PoolHealthState::Unavailable); +} + +// --------------------------------------------------------------------------- +// PoolHealthProbe — resize decisions +// --------------------------------------------------------------------------- + +#[test] +fn probe_recommends_expand_when_saturated() { + let s = pool_state("svc", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +#[test] +fn probe_recommends_expand_when_latency_over_target() { + let s = pool_state("svc", 100, 20, 80, 0, 150); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +#[test] +fn probe_recommends_expand_when_pending_requests_queued() { + let s = pool_state("svc", 100, 50, 50, 3, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +#[test] +fn probe_no_expand_when_already_at_configured_max() { + let config = PoolSizingConfig { + max_pool_size: 100, + ..default_config() + }; + let s = pool_state("svc", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &config, 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); +} + +#[test] +fn probe_recommends_shrink_when_underutilised() { + // 10% utilisation < 30% under-utilisation threshold + let s = pool_state("svc", 100, 10, 80, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::Shrink { delta: 1 }); +} + +#[test] +fn probe_no_shrink_when_already_at_configured_min() { + let config = PoolSizingConfig { + min_pool_size: 100, + ..default_config() + }; + let s = pool_state("svc", 100, 10, 80, 0, 40); + let report = PoolHealthProbe::probe(&s, &config, 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); +} + +#[test] +fn probe_no_change_in_healthy_mid_range_utilisation() { + // 60% utilisation — above under-utilisation (30%) but below saturation (90%) + let s = pool_state("svc", 100, 60, 40, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), 0, None, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); +} + +#[test] +fn probe_cooldown_suppresses_resize_within_window() { + let now = 500u64; + let last_at = Some(now - 20); // 20 s ago < 30 s cooldown + let s = pool_state("svc", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), now, last_at, 0); + assert_eq!(report.resize_decision, ResizeDecision::NoChange); +} + +#[test] +fn probe_allows_resize_after_cooldown_expires() { + let now = 500u64; + let last_at = Some(now - 31); // 31 s ago — cooldown elapsed + let s = pool_state("svc", 100, 90, 10, 0, 40); + let report = PoolHealthProbe::probe(&s, &default_config(), now, last_at, 0); + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +// --------------------------------------------------------------------------- +// PoolHealthProbe — dashboard snapshot +// --------------------------------------------------------------------------- + +#[test] +fn dashboard_snapshot_counts_pools_and_aggregates_metrics() { + let states = vec![ + pool_state("a", 100, 90, 10, 0, 40), // saturated → Warning + pool_state("b", 100, 10, 80, 0, 40), // low util → Healthy (shrink candidate) + pool_state("c", 100, 60, 40, 2, 40), // pending → Warning + ]; + let snap = PoolHealthProbe::dashboard_snapshot(&states, &default_config(), 0); + + assert_eq!(snap.pools_tracked, 3); + assert_eq!(snap.total_active_connections, 160); + assert_eq!(snap.max_pool_utilisation_bps, 9_000); + assert_eq!(snap.unhealthy_pools, 2); // a and c + assert_eq!(snap.critical_pools, 0); + assert_eq!(snap.p99_target_ms, CRITICAL_PATH_P99_MS); + assert_eq!(snap.availability_target_bps, AVAILABILITY_TARGET_BPS); +} + +#[test] +fn dashboard_snapshot_empty_returns_zeroed_metrics() { + let snap = PoolHealthProbe::dashboard_snapshot(&[], &default_config(), 0); + assert_eq!(snap.pools_tracked, 0); + assert_eq!(snap.total_active_connections, 0); + assert_eq!(snap.unhealthy_pools, 0); + assert_eq!(snap.critical_pools, 0); +} + +#[test] +fn dashboard_snapshot_counts_unavailable_pool_as_critical() { + let states = vec![pool_state("gone", 0, 0, 0, 0, 0)]; + let snap = PoolHealthProbe::dashboard_snapshot(&states, &default_config(), 0); + assert_eq!(snap.critical_pools, 1); + assert_eq!(snap.unhealthy_pools, 1); +} + +// --------------------------------------------------------------------------- +// PoolCanaryAnalysis +// --------------------------------------------------------------------------- + +#[test] +fn canary_computes_success_rate_in_basis_points() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 9_999, + p99_acquire_ms: 80, + security_review_passed: true, + }; + assert_eq!(canary.success_rate_bps(), 9_999); + assert!(canary.passes_release_gate().is_ok()); +} + +#[test] +fn canary_yields_zero_rate_when_no_acquisitions() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 0, + acquisitions_succeeded: 0, + p99_acquire_ms: 0, + security_review_passed: true, + }; + assert_eq!(canary.success_rate_bps(), 0); +} + +#[test] +fn canary_gate_fails_without_security_review() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 10_000, + p99_acquire_ms: 50, + security_review_passed: false, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::SecurityReviewRequired) + ); +} + +#[test] +fn canary_gate_fails_when_p99_exceeds_target() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 10_000, + p99_acquire_ms: 101, + security_review_passed: true, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::CanaryFailed) + ); +} + +#[test] +fn canary_gate_fails_when_success_rate_below_threshold() { + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 9_990, + p99_acquire_ms: 50, + security_review_passed: true, + }; + assert_eq!( + canary.passes_release_gate(), + Err(PoolProbeError::CanaryFailed) + ); +} + +#[test] +fn canary_gate_passes_at_boundary_conditions() { + // Exactly 9_999 / 10_000 acquisitions succeeded and P99 == 100 ms. + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 10_000, + acquisitions_succeeded: 9_999, + p99_acquire_ms: 100, + security_review_passed: true, + }; + assert!(canary.passes_release_gate().is_ok()); +} + +// --------------------------------------------------------------------------- +// PoolAdaptiveSizer +// --------------------------------------------------------------------------- + +#[test] +fn sizer_records_resize_timestamp_after_expand() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool_state("pay", 100, 90, 10, 0, 40); + let report = sizer.recommend_resize(&s, 2_000); + + assert_eq!(report.resize_decision, ResizeDecision::Expand { delta: 1 }); + assert_eq!(sizer.last_resize_at("pay"), Some(2_000)); +} + +#[test] +fn sizer_enforces_cooldown_between_consecutive_resizes() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool_state("pay", 100, 90, 10, 0, 40); + + let r1 = sizer.recommend_resize(&s, 1_000); + assert_eq!(r1.resize_decision, ResizeDecision::Expand { delta: 1 }); + + // 29 s later — cooldown not yet elapsed. + let r2 = sizer.recommend_resize(&s, 1_029); + assert_eq!(r2.resize_decision, ResizeDecision::NoChange); +} + +#[test] +fn sizer_allows_resize_after_cooldown_expires() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool_state("pay", 100, 90, 10, 0, 40); + + sizer.recommend_resize(&s, 1_000); + let r = sizer.recommend_resize(&s, 1_031); // 31 s later + assert_eq!(r.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +#[test] +fn sizer_reset_cooldown_enables_immediate_resize() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool_state("pay", 100, 90, 10, 0, 40); + + sizer.recommend_resize(&s, 1_000); + sizer.reset_cooldown("pay"); + + let r = sizer.recommend_resize(&s, 1_001); + assert_eq!(r.resize_decision, ResizeDecision::Expand { delta: 1 }); +} + +#[test] +fn sizer_accumulates_consecutive_unhealthy_count() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + // Saturated pool → Warning every probe. + let s = pool_state("svc", 100, 90, 10, 0, 40); + + for i in 0..DEGRADED_PROBE_THRESHOLD { + sizer.recommend_resize(&s, i as u64 * 100); + } + assert_eq!(sizer.consecutive_unhealthy("svc"), DEGRADED_PROBE_THRESHOLD); +} + +#[test] +fn sizer_reset_health_counters_clears_unhealthy_state() { + let mut sizer = PoolAdaptiveSizer::new(default_config()); + let s = pool_state("svc", 100, 90, 10, 0, 40); + + for i in 0..DEGRADED_PROBE_THRESHOLD + 1 { + sizer.recommend_resize(&s, i as u64 * 100); + } + sizer.reset_health_counters("svc"); + assert_eq!(sizer.consecutive_unhealthy("svc"), 0); +} + +#[test] +fn sizer_canary_gate_accepts_passing_analysis() { + let sizer = PoolAdaptiveSizer::default(); + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 5_000, + acquisitions_succeeded: 5_000, + p99_acquire_ms: 90, + security_review_passed: true, + }; + assert!(sizer.canary_gate(&canary).is_ok()); +} + +#[test] +fn sizer_canary_gate_rejects_low_success_rate() { + let sizer = PoolAdaptiveSizer::default(); + let canary = PoolCanaryAnalysis { + acquisitions_attempted: 5_000, + acquisitions_succeeded: 4_000, + p99_acquire_ms: 80, + security_review_passed: true, + }; + assert_eq!( + sizer.canary_gate(&canary), + Err(PoolProbeError::CanaryFailed) + ); +} + +// --------------------------------------------------------------------------- +// ConnectionPoolRegistry +// --------------------------------------------------------------------------- + +#[test] +fn registry_upserts_pool_and_probes_it() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool_state("payments", 50, 45, 5, 0, 40)) + .unwrap(); + + let reports = registry.probe_all(0); + assert_eq!(reports.len(), 1); + assert_eq!(reports[0].service, "payments"); +} + +#[test] +fn registry_upsert_replaces_existing_pool() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool_state("api", 100, 90, 10, 0, 40)) + .unwrap(); + // Replace with a healthy, low-utilisation snapshot. + registry + .upsert_pool(pool_state("api", 100, 5, 90, 0, 40)) + .unwrap(); + + let snap = registry.dashboard_snapshot(0); + assert_eq!(snap.total_active_connections, 5); + assert_eq!(snap.critical_pools, 0); +} + +#[test] +fn registry_dashboard_aggregates_all_pools() { + let mut registry = ConnectionPoolRegistry::default(); + registry + .upsert_pool(pool_state("svc1", 100, 90, 10, 0, 40)) + .unwrap(); + registry + .upsert_pool(pool_state("svc2", 50, 5, 40, 0, 40)) + .unwrap(); + + let snap = registry.dashboard_snapshot(0); + assert_eq!(snap.pools_tracked, 2); + assert_eq!(snap.total_active_connections, 95); +} + +#[test] +fn registry_pool_state_returns_none_for_unregistered_service() { + let registry = ConnectionPoolRegistry::default(); + assert!(registry.pool_state("ghost").is_none()); +} + +#[test] +fn registry_probe_all_returns_one_report_per_registered_pool() { + let mut registry = ConnectionPoolRegistry::default(); + for i in 0..5u32 { + registry + .upsert_pool(pool_state(&alloc::format!("svc{i}"), 100, 50, 50, 0, 40)) + .unwrap(); + } + let reports = registry.probe_all(0); + assert_eq!(reports.len(), 5); +} + +#[test] +fn registry_rejects_new_pool_when_at_capacity() { + let mut registry = ConnectionPoolRegistry::new(default_config()); + // Fill the registry to capacity. + for i in 0..MAX_SERVICE_POOLS { + registry + .upsert_pool(pool_state(&alloc::format!("svc{i}"), 10, 5, 5, 0, 40)) + .unwrap(); + } + // One more new service must be rejected. + let result = registry.upsert_pool(pool_state("overflow", 10, 5, 5, 0, 40)); + assert_eq!(result, Err(PoolProbeError::TooManyPools)); +} + +// Module-level use for alloc::format in the test above. +extern crate alloc;