From 5120ea313c5a10a860885d5e4a6bce54d172020c Mon Sep 17 00:00:00 2001 From: Pristley Date: Sat, 4 Jul 2026 04:35:49 +0000 Subject: [PATCH 1/5] docs: update changelog with degradation detector and similarity registry --- docs/internal/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/internal/CHANGELOG.md b/docs/internal/CHANGELOG.md index 92f58aa..de85605 100644 --- a/docs/internal/CHANGELOG.md +++ b/docs/internal/CHANGELOG.md @@ -64,6 +64,13 @@ Release entries are maintained automatically by the CD workflow on tagged releas - Automatic validation: Ensures first 3 windows in descending order to prevent alert storms - Comprehensive test suite: 5 test suites validating threshold calculations, error budget %, config structure, duration parsing, and real-world scenarios - New guide: `docs/guides/multi-burn-rate-alerting.md` with theory, configuration examples, real incident timelines, and tuning guidelines + - **Degradation Detection & Similarity Scoring (Experimental)** + - Added `src/degradation.rs`: ADWIN-like gradual drift detector, Z-score spike detector, oscillation heuristic, and `DegradationDetector` API (`add_metric`, `detect_change`, `reset`, `exponential_histogram`). + - Added `DegradationType` and `DegradationEvent` (serde serializable) for structured degradation reporting. + - Added comprehensive unit tests in `tests/degradation_tests.rs` covering gradual drift, sudden spikes, oscillation, NaN handling, window behavior, and property tests via `proptest`. + - Added `src/similarity_traits.rs`: `SimilarityScorer` trait and `ScorerRegistry` with thread-safe registry and global `default_registry()`. + - Added example scorer `src/similarity_implementations.rs::DummyScorer` and tests `tests/similarity_registry_tests.rs`. + - Added GitHub Actions workflow `.github/workflows/rust.yml` to run `cargo build` and `cargo test` on push/PR. - **Documentation Organization** - Created `docs/cli/` directory with all CLI documentation From 6297cb864d8c8068b48738111275b9942b943215 Mon Sep 17 00:00:00 2001 From: Pristley Date: Sat, 4 Jul 2026 04:42:42 +0000 Subject: [PATCH 2/5] feat: add Bert/Roberta/Cosine scorers with LRU cache and tests --- Cargo.toml | 4 +- src/similarity_implementations.rs | 273 +++++++++++++++++++++++++++++- 2 files changed, 272 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 20cff95..84d748f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,8 @@ rayon = "1.7" clap = { version = "4.5", features = ["derive"] } anyhow = "1.0" tokio = { version = "1.40", features = ["full"] } -reqwest = { version = "0.11", features = ["json"] } +reqwest = { version = "0.11", features = ["json", "blocking"] } +lru = "0.9" redis = { version = "0.25", features = ["tokio-comp", "connection-manager"] } sha2 = "0.10" base64 = "0.22" @@ -37,6 +38,7 @@ criterion = "0.5" proptest = "1.6" rand = "0.8" rand_distr = "0.4" +mockito = "1.0" [[bench]] name = "composite_slo_dag" diff --git a/src/similarity_implementations.rs b/src/similarity_implementations.rs index 96d5b72..1318414 100644 --- a/src/similarity_implementations.rs +++ b/src/similarity_implementations.rs @@ -1,10 +1,243 @@ -use crate::{Result}; +use crate::Result; +use log::{debug, warn}; +use lru::LruCache; +use reqwest::blocking::Client; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; use crate::similarity_traits::SimilarityScorer; -/// A tiny example scorer used for tests and examples. +fn hash_to_string(a: &str, b: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(a.as_bytes()); + hasher.update(b.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +/// Stateless cosine-like scorer using simple word-overlap vectors. +pub struct CosineSimScorer; + +impl CosineSimScorer { + pub fn new() -> Self { + Self {} + } + + fn tokenize(s: &str) -> HashMap { + let mut m = HashMap::new(); + for w in s + .split(|c: char| !c.is_alphanumeric()) + .filter(|t| !t.is_empty()) + { + *m.entry(w.to_lowercase()).or_insert(0.0) += 1.0; + } + m + } + + fn cosine(a: &HashMap, b: &HashMap) -> f64 { + let mut dot = 0.0; + let mut na = 0.0; + let mut nb = 0.0; + for (k, va) in a.iter() { + na += va * va; + if let Some(vb) = b.get(k) { + dot += va * vb; + } + } + for vb in b.values() { + nb += vb * vb; + } + if na == 0.0 || nb == 0.0 { + 0.0 + } else { + dot / (na.sqrt() * nb.sqrt()) + } + } +} + +impl SimilarityScorer for CosineSimScorer { + fn score(&self, reference: &str, generated: &str) -> Result { + let a = CosineSimScorer::tokenize(reference); + let b = CosineSimScorer::tokenize(generated); + Ok(CosineSimScorer::cosine(&a, &b)) + } + + fn batch_score(&self, pairs: Vec<(&str, &str)>) -> Result> { + let mut out = Vec::with_capacity(pairs.len()); + for (r, g) in pairs { + out.push(self.score(r, g)?); + } + Ok(out) + } + + fn name(&self) -> &'static str { + "cosine_simple" + } +} + +impl Clone for CosineSimScorer { + fn clone(&self) -> Self { + Self::new() + } +} + +/// HTTP-backed scorer with LRU cache. Falls back to `CosineSimScorer` on errors. +pub struct BertScorer { + pub http_endpoint: String, + pub model_name: String, + pub timeout_secs: u64, + pub cache: Arc>>, + pub cache_hits: AtomicU64, + client: Client, + fallback: CosineSimScorer, +} + +impl BertScorer { + pub fn new(endpoint: String, model: String) -> Self { + let client = Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .unwrap(); + Self { + http_endpoint: endpoint, + model_name: model, + timeout_secs: 5, + cache: Arc::new(Mutex::new(LruCache::new(1000))), + cache_hits: AtomicU64::new(0), + client, + fallback: CosineSimScorer::new(), + } + } + + fn fetch_similarity(&self, reference: &str, generated: &str) -> Result { + let payload = serde_json::json!({"text1": reference, "text2": generated, "model": self.model_name}); + let resp = self.client.post(&self.http_endpoint).json(&payload).send(); + match resp { + Ok(r) => { + let v: Value = r.json().map_err(|e| crate::NeuralBudgetError::FormatError(format!("invalid json: {}", e)))?; + if let Some(sim) = v.get("similarity") { + if let Some(f) = sim.as_f64() { + return Ok(f); + } + } + Err(crate::NeuralBudgetError::FormatError("missing similarity field".to_string())) + } + Err(e) => Err(crate::NeuralBudgetError::EvaluationError(format!("http error: {}", e))), + } + } +} + +impl SimilarityScorer for BertScorer { + fn score(&self, reference: &str, generated: &str) -> Result { + let key = hash_to_string(reference, generated); + if let Ok(mut c) = self.cache.lock() { + if let Some(v) = c.get(&key) { + self.cache_hits.fetch_add(1, Ordering::SeqCst); + debug!("cache hit for key {}", key); + return Ok(*v); + } + } + + match self.fetch_similarity(reference, generated) { + Ok(sim) => { + if let Ok(mut c) = self.cache.lock() { + c.put(key.clone(), sim); + } + Ok(sim) + } + Err(e) => { + warn!("BertScorer http failed {}, falling back to cosine", e); + let f = self.fallback.score(reference, generated)?; + if let Ok(mut c) = self.cache.lock() { + c.put(key, f); + } + Ok(f) + } + } + } + + fn batch_score(&self, pairs: Vec<(&str, &str)>) -> Result> { + let mut out = Vec::with_capacity(pairs.len()); + for (r, g) in pairs { + out.push(self.score(r, g)?); + } + Ok(out) + } + + fn name(&self) -> &'static str { + "bert_scorer" + } + + fn model_version(&self) -> Option { + Some(self.model_name.clone()) + } + + fn cache_hits(&self) -> Option { + Some(self.cache_hits.load(Ordering::SeqCst)) + } +} + +impl Clone for BertScorer { + fn clone(&self) -> Self { + Self { + http_endpoint: self.http_endpoint.clone(), + model_name: self.model_name.clone(), + timeout_secs: self.timeout_secs, + cache: Arc::clone(&self.cache), + cache_hits: AtomicU64::new(self.cache_hits.load(Ordering::SeqCst)), + client: self.client.clone(), + fallback: self.fallback.clone(), + } + } +} + +/// RobertaScorer mirrors BertScorer but with a different default model name. +pub struct RobertaScorer { + inner: BertScorer, +} + +impl RobertaScorer { + pub fn new(endpoint: String) -> Self { + Self { + inner: BertScorer::new(endpoint, "all-roberta-large-v1".to_string()), + } + } +} + +impl SimilarityScorer for RobertaScorer { + fn score(&self, reference: &str, generated: &str) -> Result { + self.inner.score(reference, generated) + } + + fn batch_score(&self, pairs: Vec<(&str, &str)>) -> Result> { + self.inner.batch_score(pairs) + } + + fn name(&self) -> &'static str { + "roberta_scorer" + } + + fn model_version(&self) -> Option { + self.inner.model_version() + } + + fn cache_hits(&self) -> Option { + self.inner.cache_hits() + } +} + +impl Clone for RobertaScorer { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +/// Keep the previous DummyScorer available for examples/tests. pub struct DummyScorer { version: Option, cache_hits: AtomicU64, @@ -21,7 +254,6 @@ impl DummyScorer { impl SimilarityScorer for DummyScorer { fn score(&self, reference: &str, generated: &str) -> Result { - // trivial scoring: exact matches -> 1.0, otherwise 0.5 if reference == generated { Ok(1.0) } else { @@ -53,3 +285,36 @@ impl SimilarityScorer for DummyScorer { pub fn arc_dummy() -> Arc { Arc::new(DummyScorer::new()) } + +#[cfg(test)] +mod tests { + use super::*; + use mockito::mock; + + #[test] + fn test_bert_cache_and_fallback() { + let _m = mock("POST", "/embed") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"similarity": 0.85}"#) + .create(); + + let endpoint = format!("{}{}", &mockito::server_url(), "/embed"); + let scorer = BertScorer::new(endpoint, "mymodel".to_string()); + let res = scorer.score("hello world", "hello world").unwrap(); + assert!(res > 0.8 && res <= 1.0); + + // second call should hit cache + let res2 = scorer.score("hello world", "hello world").unwrap(); + assert_eq!(res, res2); + } + + #[test] + fn test_fetch_failure_fallback() { + // point to a non-routable address to force error and fallback + let scorer = BertScorer::new("http://127.0.0.1:9/embed".to_string(), "mymodel".to_string()); + let res = scorer.score("a quick fox", "the quick fox").unwrap(); + // cosine fallback should return > 0 + assert!(res >= 0.0 && res <= 1.0); + } +} From e6cb5acb6ac24c80bd89f13f5c58a16d74b36b86 Mon Sep 17 00:00:00 2001 From: Pristley Date: Sat, 4 Jul 2026 04:46:51 +0000 Subject: [PATCH 3/5] feat: add DynamicCostCalculator and integrate with LLM evaluator --- src/cost_slo.rs | 156 +++++++++++++++++++++++++++++++++++++++++ src/genai_evaluator.rs | 22 +++++- 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/src/cost_slo.rs b/src/cost_slo.rs index c049260..8706669 100644 --- a/src/cost_slo.rs +++ b/src/cost_slo.rs @@ -5,6 +5,162 @@ use crate::NeuralBudgetError; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +/// Token pricing configuration for a model. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TokenCostConfig { + pub model: String, + pub input_cost_per_1k: f64, + pub output_cost_per_1k: f64, + pub pricing_date: u64, + pub currency: String, +} + +impl TokenCostConfig { + pub fn new(model: String, input_cost_per_1k: f64, output_cost_per_1k: f64, pricing_date: u64, currency: String) -> Self { + Self { model, input_cost_per_1k, output_cost_per_1k, pricing_date, currency } + } +} + +/// Errors specific to dynamic cost calculator. +#[derive(Debug)] +pub enum TokenCostError { + ModelNotFound(String), + InvalidTokenCount(String), +} + +impl From for NeuralBudgetError { + fn from(e: TokenCostError) -> Self { + match e { + TokenCostError::ModelNotFound(m) => NeuralBudgetError::ConfigError(format!("model not found: {}", m)), + TokenCostError::InvalidTokenCount(s) => NeuralBudgetError::EvaluationError(format!("invalid token count: {}", s)), + } + } +} + +/// Breakdown of costs for reporting. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CostMetrics { + pub cost_breakdown: HashMap, + pub model_used: String, + pub pricing_date: u64, +} + +impl CostMetrics { + pub fn new(model_used: String, pricing_date: u64, input_cost: f64, output_cost: f64) -> Self { + let mut map = HashMap::new(); + map.insert("input".to_string(), input_cost); + map.insert("output".to_string(), output_cost); + map.insert("total".to_string(), input_cost + output_cost); + Self { cost_breakdown: map, model_used, pricing_date } + } +} + +/// Dynamic cost calculator holding pricing configs. +#[derive(Clone)] +pub struct DynamicCostCalculator { + configs: Arc>>, + last_updated: Arc>, +} + +impl DynamicCostCalculator { + pub fn new() -> Self { + Self { configs: Arc::new(RwLock::new(HashMap::new())), last_updated: Arc::new(RwLock::new(0)) } + } + + /// Update pricing for a model (overwrites existing entry). + pub fn update_pricing(&self, model: String, config: TokenCostConfig) -> Result<(), NeuralBudgetError> { + let mut cfg = self.configs.write().map_err(|e| NeuralBudgetError::EvaluationError(format!("lock poisoned: {}", e)))?; + cfg.insert(model.clone(), config.clone()); + let mut lu = self.last_updated.write().map_err(|e| NeuralBudgetError::EvaluationError(format!("lock poisoned: {}", e)))?; + *lu = config.pricing_date; + Ok(()) + } + + /// Get pricing config for a model; exact match preferred, then wildcard. + pub fn get_pricing(&self, model: &str) -> Result { + let cfg = self.configs.read().map_err(|e| NeuralBudgetError::EvaluationError(format!("lock poisoned: {}", e)))?; + if let Some(c) = cfg.get(model) { + return Ok(c.clone()); + } + // wildcard matching: prefer longest prefix/suffix match + // patterns stored as keys may include '*' + // exact match failed; search patterns + let mut best: Option<(usize, TokenCostConfig)> = None; + for (k, v) in cfg.iter() { + if k.contains('*') { + // convert simple wildcard to prefix/suffix checks + if wildcard_match(k, model) { + let score = k.len(); + if best.is_none() || score > best.as_ref().unwrap().0 { + best = Some((score, v.clone())); + } + } + } + } + if let Some((_, c)) = best { + return Ok(c); + } + Err(TokenCostError::ModelNotFound(model.to_string()).into()) + } + + /// Calculate cost for given model and token counts. + pub fn calculate_cost(&self, model: &str, input_tokens: u64, output_tokens: u64) -> Result { + if input_tokens as i128 < 0 || output_tokens as i128 < 0 { + return Err(TokenCostError::InvalidTokenCount("negative token count".to_string()).into()); + } + let cfg = self.get_pricing(model)?; + let input_cost = (input_tokens as f64 / 1000.0) * cfg.input_cost_per_1k; + let output_cost = (output_tokens as f64 / 1000.0) * cfg.output_cost_per_1k; + Ok(input_cost + output_cost) + } + + /// List known models (keys) + pub fn list_models(&self) -> Vec { + if let Ok(cfg) = self.configs.read() { + cfg.keys().cloned().collect() + } else { + Vec::new() + } + } +} + +/// Simple wildcard matcher supporting '*' anywhere. +fn wildcard_match(pattern: &str, text: &str) -> bool { + if pattern == "*" { + return true; + } + let parts: Vec<&str> = pattern.split('*').collect(); + if parts.len() == 1 { + return pattern == text; + } + let mut idx = 0usize; + // if pattern doesn't start with *, first part must match at start + if !pattern.starts_with('*') { + if !text.starts_with(parts[0]) { + return false; + } + idx = parts[0].len(); + } + for (i, part) in parts.iter().enumerate() { + if part.is_empty() { continue; } + if i == parts.len() - 1 && !pattern.ends_with('*') { + // last part must match at end + if !text.ends_with(part) { + return false; + } + } else { + if let Some(pos) = text[idx..].find(part) { + idx += pos + part.len(); + } else { + return false; + } + } + } + true +} /// Cost budget configuration for input/output tokens. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] diff --git a/src/genai_evaluator.rs b/src/genai_evaluator.rs index 01588ae..fda235d 100644 --- a/src/genai_evaluator.rs +++ b/src/genai_evaluator.rs @@ -34,6 +34,7 @@ use crate::{NeuralBudgetError, Result}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::sync::Arc; +use crate::cost_slo::DynamicCostCalculator; /// LLM provider configuration #[derive(Debug, Clone, Serialize, Deserialize)] @@ -126,6 +127,7 @@ pub struct LlmJudgeEvaluator { pub cache_config: Option, cache_client: Option>, http_client: reqwest::Client, + cost_calculator: Option>, } impl LlmJudgeEvaluator { @@ -137,9 +139,16 @@ impl LlmJudgeEvaluator { cache_config: None, cache_client: None, http_client: reqwest::Client::new(), + cost_calculator: None, } } + /// Attach a dynamic cost calculator to compute per-request token costs. + pub fn with_cost_calculator(mut self, calc: Arc) -> Self { + self.cost_calculator = Some(calc); + self + } + /// Add Redis caching to the evaluator pub async fn with_redis_cache(mut self, redis_url: &str, ttl_seconds: u64) -> Result { let client = redis::Client::open(redis_url).map_err(|e| { @@ -278,7 +287,18 @@ impl LlmJudgeEvaluator { dimension_scores, weighted_score, pass, - total_cost_usd: total_cost, + total_cost_usd: { + // If a cost calculator is attached, prefer token-based cost calculation + if let Some(calc) = &self.cost_calculator { + // best-effort: treat all tokens as output tokens if input unknown + match calc.calculate_cost(self.provider.model(), 0, total_tokens as u64) { + Ok(c) => c, + Err(_) => total_cost, + } + } else { + total_cost + } + }, total_tokens, }; From a351b208bb6ca776cebab404bd4b17416a09dd46 Mon Sep 17 00:00:00 2001 From: Pristley Date: Sat, 4 Jul 2026 04:50:18 +0000 Subject: [PATCH 4/5] feat: add advanced hallucination detector and metrics --- src/hallucination_advanced.rs | 272 ++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + 2 files changed, 274 insertions(+) create mode 100644 src/hallucination_advanced.rs diff --git a/src/hallucination_advanced.rs b/src/hallucination_advanced.rs new file mode 100644 index 0000000..fc484c9 --- /dev/null +++ b/src/hallucination_advanced.rs @@ -0,0 +1,272 @@ +//! Advanced hallucination detection utilities. +//! +//! Provides lightweight, rule-based approximations for entailment, contradiction, +//! and ungroundedness detection suitable for integration into SLO pipelines. + +use crate::{NeuralBudgetError, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +/// Types of hallucination detected. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum HallucinationScore { + Entailment, + Contradiction, + Ungroundedness, + HighConfidence, + FactuallyCorrect, +} + +/// Result metrics from hallucination analysis. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HallucinationMetrics { + pub factual_consistency: f64, // 0.0..1.0 + pub hallucination_types: Vec, + pub confidence: f64, // 0.0..1.0 + pub explanation: String, + pub conflicting_entities: Vec, +} + +impl HallucinationMetrics { + pub fn new() -> Self { + Self { + factual_consistency: 1.0, + hallucination_types: Vec::new(), + confidence: 1.0, + explanation: String::new(), + conflicting_entities: Vec::new(), + } + } +} + +/// Hallucination detector with optional RAG context and entity extractor. +pub struct HallucinationDetector { + reference: String, + rag_context: Option, + entity_extractor: Option Vec>>, +} + +impl HallucinationDetector { + /// Create a new detector with a reference text. + pub fn new(reference: &str) -> Self { + Self { + reference: reference.to_string(), + rag_context: None, + entity_extractor: None, + } + } + + /// Set RAG context used for grounding checks. + pub fn set_rag_context(&mut self, ctx: &str) { + self.rag_context = Some(ctx.to_string()); + } + + /// Set a custom entity extractor function. If not set, a basic heuristic is used. + pub fn set_entity_extractor(&mut self, f: F) + where + F: 'static + Fn(&str) -> Vec, + { + self.entity_extractor = Some(Box::new(f)); + } + + /// Analyze a generated text and produce hallucination metrics. + pub fn analyze(&self, generated: &str) -> Result { + if self.reference.trim().is_empty() { + return Err(NeuralBudgetError::ConfigError( + "reference text required for analysis".to_string(), + )); + } + + let mut metrics = HallucinationMetrics::new(); + + // extract entities + let gen_entities = self.extract_entities(generated); + let ref_entities = self.extract_entities(&self.reference); + let rag_entities = if let Some(ctx) = &self.rag_context { + self.extract_entities(ctx) + } else { + Vec::new() + }; + + // contradictions + let contradictions = self.detect_contradictions(&ref_entities, &gen_entities, generated); + if !contradictions.is_empty() { + metrics.hallucination_types.push(HallucinationScore::Contradiction); + metrics.conflicting_entities = contradictions.clone(); + } + + // ungroundedness: entities in generated not present in ref or RAG + let missing = self.detect_ungroundedness(&gen_entities, &ref_entities, &rag_entities); + if !missing.is_empty() { + metrics.hallucination_types.push(HallucinationScore::Ungroundedness); + } + + // entailment approximation + let entailment_score = self.score_entailment(&ref_entities, &gen_entities, generated); + if entailment_score > 0.7 && contradictions.is_empty() && missing.is_empty() { + metrics.hallucination_types.push(HallucinationScore::Entailment); + } + + // high confidence heuristic: if text contains hedging words, reduce confidence + let mut confidence = 1.0_f64; + if generated.to_lowercase().contains("i think") + || generated.to_lowercase().contains("maybe") + || generated.to_lowercase().contains("might") + { + confidence *= 0.8; + } + + // compute hallucination score: contradictions penalize more than ungroundedness + let contradiction_penalty = (metrics.conflicting_entities.len() as f64) * 0.7; + let ungrounded_penalty = (missing.len() as f64) * 0.3; + let raw_penalty = (contradiction_penalty + ungrounded_penalty).min(1.0); + let factual_consistency = ((1.0 - raw_penalty) * confidence).clamp(0.0, 1.0); + + // explanation + let mut explanation = Vec::new(); + if !metrics.conflicting_entities.is_empty() { + explanation.push(format!( + "Contradictions on: {}", + metrics.conflicting_entities.join(", ") + )); + } + if !missing.is_empty() { + explanation.push(format!("Ungrounded entities: {}", missing.join(", "))); + } + if explanation.is_empty() { + explanation.push("No hallucinations detected".to_string()); + metrics.hallucination_types.push(HallucinationScore::FactuallyCorrect); + } + + metrics.factual_consistency = factual_consistency; + metrics.confidence = confidence; + metrics.explanation = explanation.join("; "); + + Ok(metrics) + } + + /// Detect contradictions between reference entities and generated entities. + fn detect_contradictions(&self, ref_entities: &[String], gen_entities: &[String], generated_text: &str) -> Vec { + let mut conflicts = Vec::new(); + let negations = vec!["not", "no", "never", "doesn't", "does not", "didn't", "without"]; + let gen_lower = generated_text.to_lowercase(); + let ref_set: HashSet = ref_entities.iter().map(|s| s.to_lowercase()).collect(); + + for ge in gen_entities { + let gl = ge.to_lowercase(); + if ref_set.contains(&gl) { + // look for negation near entity in generated text + for neg in &negations { + let pattern1 = format!("{} {}", neg, gl); + let pattern2 = format!("{} {}", gl, neg); + if gen_lower.contains(&pattern1) || gen_lower.contains(&pattern2) { + conflicts.push(ge.clone()); + break; + } + } + } + } + conflicts + } + + /// Detect ungrounded entities present in generated but not in reference or RAG context. + fn detect_ungroundedness(&self, gen_entities: &[String], ref_entities: &[String], rag_entities: &[String]) -> Vec { + let ref_set: HashSet = ref_entities.iter().map(|s| s.to_lowercase()).collect(); + let rag_set: HashSet = rag_entities.iter().map(|s| s.to_lowercase()).collect(); + let mut missing = Vec::new(); + for ge in gen_entities { + let gl = ge.to_lowercase(); + if !ref_set.contains(&gl) && !rag_set.contains(&gl) { + missing.push(ge.clone()); + } + } + missing + } + + /// Approximate entailment score between reference and generated. + fn score_entailment(&self, ref_entities: &[String], gen_entities: &[String], generated_text: &str) -> f64 { + let ref_set: HashSet = ref_entities.iter().map(|s| s.to_lowercase()).collect(); + let gen_set: HashSet = gen_entities.iter().map(|s| s.to_lowercase()).collect(); + let common = ref_set.intersection(&gen_set).count() as f64; + let ref_count = ref_set.len() as f64; + let entity_score = if ref_count > 0.0 { common / ref_count } else { 0.0 }; + + // token overlap + let ref_tokens: HashSet = self.token_set(&self.reference); + let gen_tokens: HashSet = self.token_set(generated_text); + let inter = ref_tokens.intersection(&gen_tokens).count() as f64; + let union = ref_tokens.union(&gen_tokens).count() as f64; + let token_overlap = if union > 0.0 { inter / union } else { 0.0 }; + + // weighted combination + (0.6 * entity_score) + (0.4 * token_overlap) + } + + /// Extract entities using either the provided extractor or a simple heuristic. + fn extract_entities(&self, text: &str) -> Vec { + if let Some(extractor) = &self.entity_extractor { + return extractor(text); + } + // naive extractor: words starting with uppercase and length>1 + text.split_whitespace() + .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric())) + .filter(|w| !w.is_empty()) + .filter(|w| { + w.chars() + .next() + .map(|c| c.is_uppercase()) + .unwrap_or(false) + }) + .map(|s| s.trim_matches(|c: char| !c.is_alphanumeric()).to_string()) + .collect() + } + + fn token_set(&self, text: &str) -> HashSet { + text.split(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_lowercase()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_entailment_all_facts_present() { + let ref_text = "Paris is the capital of France. The Eiffel Tower is in Paris."; + let gen = "The Eiffel Tower stands in Paris, the capital of France."; + let d = HallucinationDetector::new(ref_text); + let m = d.analyze(gen).unwrap(); + assert!(m.hallucination_types.contains(&HallucinationScore::Entailment)); + assert!(m.factual_consistency > 0.5); + } + + #[test] + fn test_contradiction_detection() { + let ref_text = "The patient has diabetes."; + let gen = "The patient does not have diabetes."; + let d = HallucinationDetector::new(ref_text); + let m = d.analyze(gen).unwrap(); + assert!(m.hallucination_types.contains(&HallucinationScore::Contradiction)); + assert!(m.conflicting_entities.contains(&"diabetes".to_string()) || !m.conflicting_entities.is_empty()); + } + + #[test] + fn test_ungroundedness_detection() { + let ref_text = "The company was founded in 1999 in Berlin."; + let gen = "The company was founded in 1999 in Berlin. It operates offices in Reykjavik."; + let mut d = HallucinationDetector::new(ref_text); + d.set_rag_context("Berlin is the capital of Germany."); + let m = d.analyze(gen).unwrap(); + assert!(m.hallucination_types.contains(&HallucinationScore::Ungroundedness)); + } + + #[test] + fn test_empty_reference_error() { + let d = HallucinationDetector::new(""); + let err = d.analyze("some text"); + assert!(err.is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 1dc2ae2..d95457d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ mod exporter; mod forecasting; mod genai_evaluator; mod genai_slo; +mod hallucination_advanced; mod groundedness; pub mod openslo; mod otlp; @@ -27,6 +28,7 @@ pub use genai_evaluator::*; pub use genai_slo::*; pub use groundedness::*; pub use openslo::*; +pub use hallucination_advanced::*; pub use otlp::*; pub use python::*; pub use slo_graph::*; From 2492476bb6448f5aba6fa7e265b0640583bfe27f Mon Sep 17 00:00:00 2001 From: Pristley Date: Sat, 4 Jul 2026 05:25:55 +0000 Subject: [PATCH 5/5] chore(release): prepare v0.2.0 - docs and Python convenience wrappers/examples --- README.md | 5 ++ docs/internal/CHANGELOG.md | 16 ++++++ examples/custom_similarity_scorer.py | 21 ++++++++ examples/degradation_detection.py | 21 ++++++++ python/neuralbudget/degradation.py | 51 +++++++++++++++++++ python/neuralbudget/hallucination_advanced.py | 26 ++++++++++ python/neuralbudget/similarity.py | 38 ++++++++++++++ 7 files changed, 178 insertions(+) create mode 100644 examples/custom_similarity_scorer.py create mode 100644 examples/degradation_detection.py create mode 100644 python/neuralbudget/degradation.py create mode 100644 python/neuralbudget/hallucination_advanced.py create mode 100644 python/neuralbudget/similarity.py diff --git a/README.md b/README.md index 4ffdf3e..8279770 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,11 @@ --- +## Release v0.2.0 — Advanced ML / GenAI Features + +This release adds advanced monitoring and explanation capabilities for ML and GenAI workloads: model degradation detection, pluggable semantic similarity scorers, LLM-specific metrics (token cost tracking and hallucination detection), and feature drift explanation with lightweight SHAP approximations. See [docs/advanced-mlgenai-features.md](docs/advanced-mlgenai-features.md) for details and examples. + + ## 🎯 The Problem Nobody Talks About Your Prometheus rules say availability is **99.97%**. Your incident logs say **98.2%**. Your auditor asks: "Which one is correct?" diff --git a/docs/internal/CHANGELOG.md b/docs/internal/CHANGELOG.md index de85605..3e690b3 100644 --- a/docs/internal/CHANGELOG.md +++ b/docs/internal/CHANGELOG.md @@ -72,6 +72,22 @@ Release entries are maintained automatically by the CD workflow on tagged releas - Added example scorer `src/similarity_implementations.rs::DummyScorer` and tests `tests/similarity_registry_tests.rs`. - Added GitHub Actions workflow `.github/workflows/rust.yml` to run `cargo build` and `cargo test` on push/PR. + ## [0.2.0] - 2026-07-04 + + ### Added + + - Degradation detection primitives (`src/degradation.rs`) with ADWIN-like gradual drift detection, spike detection, and oscillation heuristics. Includes `DegradationDetector`, `DegradationEvent`, and comprehensive unit tests. + - Pluggable similarity scoring API (`src/similarity_traits.rs`) and implementations (`src/similarity_implementations.rs`) including `BertScorer`, `RobertaScorer`, `CosineSimScorer`, LRU caching and HTTP fallback. + - Advanced hallucination detection utilities (`src/hallucination_advanced.rs`) for entailment, contradiction and ungroundedness heuristics. + - Feature drift analysis and explanation (`src/drift_explanation.rs`) with KS two-sample tests, top-feature ranking, and SHAP-style `ShapExplainer` (`src/shap_integration.rs`). + - Python bindings and convenience wrappers: PyO3 bindings in `src/python.rs` and convenience modules under `python/neuralbudget/` for degradation, similarity, hallucination and drift explainers. + - Prometheus exporter extensions (`src/exporter.rs`) to observe degradation, hallucination, and token cost metrics. + + ### Changed + + - Bumped package version to `0.2.0` in `Cargo.toml` and `pyproject.toml`. + + - **Documentation Organization** - Created `docs/cli/` directory with all CLI documentation - Created `docs/guides/prometheus-rule-generation.md` with comprehensive burn rate guide diff --git a/examples/custom_similarity_scorer.py b/examples/custom_similarity_scorer.py new file mode 100644 index 0000000..12f58e1 --- /dev/null +++ b/examples/custom_similarity_scorer.py @@ -0,0 +1,21 @@ +"""Example demonstrating registration and use of a similarity scorer.""" +from neuralbudget.similarity import SimilarityRegistry +import neuralbudget + + +def main(): + reg = SimilarityRegistry(default_name="dummy_default") + + # Use the built-in DummyScorer (provided by the Rust lib) for examples + dummy = neuralbudget.DummyScorer() + reg.register("dummy", dummy) + + scorer = reg.get_scorer("dummy") + score = scorer.score("The quick brown fox", "The quick brown fox") + print("Exact match score:", score) + + print("Available scorers:", reg.list_scorers()) + + +if __name__ == "__main__": + main() diff --git a/examples/degradation_detection.py b/examples/degradation_detection.py new file mode 100644 index 0000000..40487f2 --- /dev/null +++ b/examples/degradation_detection.py @@ -0,0 +1,21 @@ +"""Simple example showing the DegradationDetector convenience wrapper.""" +from neuralbudget.degradation import DegradationDetector + + +def main(): + d = DegradationDetector(window_size=60, drift_threshold=0.10, spike_threshold=2.0) + + # warm-up: stable values + for _ in range(40): + d.add_metric(50.0) + + # introduce a gradual drift + for i in range(20): + d.add_metric(50.0 + i * 1.5) + + ev = d.detect_change() + print("Degradation event:", ev) + + +if __name__ == "__main__": + main() diff --git a/python/neuralbudget/degradation.py b/python/neuralbudget/degradation.py new file mode 100644 index 0000000..9c50d43 --- /dev/null +++ b/python/neuralbudget/degradation.py @@ -0,0 +1,51 @@ +""" +Convenience wrapper for the Degradation detector exposed by the Rust extension. +This module provides a small, Pythonic adapter around the PyO3 class so callers +get plain dicts and simple methods. +""" +from typing import Optional, Dict, Any + +import neuralbudget + + +class DegradationDetector: + """Python convenience wrapper around the Rust `DegradationDetector`. + + Parameters + - window_size: rolling window size (int) + - drift_threshold: threshold used for gradual drift detection (float) + - spike_threshold: z-score multiplier for spike detection (float) + """ + + def __init__(self, window_size: int = 100, drift_threshold: float = 0.15, spike_threshold: float = 2.5): + self._inner = neuralbudget.DegradationDetector(window_size, drift_threshold, spike_threshold) + + def add_metric(self, value: float) -> None: + """Add a new metric sample to the detector.""" + return self._inner.add_metric(value) + + def detect_change(self) -> Optional[Dict[str, Any]]: + """Run detection and return either None or a dict describing the event. + + The returned dict uses plain Python primitives for easy consumption. + """ + ev = self._inner.detect_change() + if ev is None: + return None + # Convert the Rust/PyO3 object to a simple dict (best-effort attribute access) + return { + "timestamp": getattr(ev, "timestamp", None), + "degradation_type": str(getattr(ev, "degradation_type", getattr(ev, "kind", None))), + "magnitude": getattr(ev, "magnitude", None), + "confidence": getattr(ev, "confidence", None), + "duration_samples": getattr(ev, "duration_samples", None), + "affected_metric": getattr(ev, "affected_metric", None), + } + + def reset(self) -> None: + """Reset internal detector state and history.""" + return self._inner.reset() + + def history_len(self) -> int: + """Return the length of the internal history buffer.""" + return int(self._inner.history_len()) diff --git a/python/neuralbudget/hallucination_advanced.py b/python/neuralbudget/hallucination_advanced.py new file mode 100644 index 0000000..b958223 --- /dev/null +++ b/python/neuralbudget/hallucination_advanced.py @@ -0,0 +1,26 @@ +""" +Convenience helpers for hallucination/groundedness analysis. +This adapter exposes a tiny, stable surface that converts Rust results to plain +Python dicts for easier consumption in examples and notebooks. +""" +from typing import Any, Dict, List +import neuralbudget + + +def analyze_hallucination(response: str, documents: List[str]) -> Dict[str, Any]: + """Run the higher-level hallucination analysis and return a dict. + + This is a convenience wrapper that uses the library's default detector. + """ + # There is a Rust-side `HallucinationDetector` that returns a struct of + # metrics; call that and convert to a dict. + detector = neuralbudget.HallucinationDetector() + metrics = detector.analyze(response, documents) + + return { + "factual_consistency": getattr(metrics, "factual_consistency", None), + "hallucination_rate": getattr(metrics, "hallucination_rate", None), + "grounded_count": getattr(metrics, "grounded_count", None), + "hallucinated_count": getattr(metrics, "hallucinated_count", None), + "summary": getattr(metrics, "summary", None), + } diff --git a/python/neuralbudget/similarity.py b/python/neuralbudget/similarity.py new file mode 100644 index 0000000..a77a6c1 --- /dev/null +++ b/python/neuralbudget/similarity.py @@ -0,0 +1,38 @@ +""" +Convenience helpers for the similarity scorer registry and scorers. +This module wraps the low-level PyO3 objects to present a compact Python API. +""" +from typing import List +import neuralbudget + + +class SimilarityRegistry: + """Thin wrapper around the Rust-backed `SimilarityRegistry`. + + Example: + reg = SimilarityRegistry("dummy_default") + reg.register("dummy", neuralbudget.DummyScorer()) + s = reg.get_scorer("dummy") + print(s.score("a","b")) + """ + + def __init__(self, default_name: str = "default"): + self._inner = neuralbudget.SimilarityRegistry(default_name) + + def register(self, name: str, scorer) -> None: + return self._inner.register(name, scorer) + + def get_scorer(self, name: str): + return self._inner.get_scorer(name) + + def list_scorers(self) -> List[str]: + return list(self._inner.list_scorers()) + + def set_default(self, name: str) -> None: + return self._inner.set_default(name) + + def default_name(self) -> str: + return str(self._inner.default_name()) + + def remove_scorer(self, name: str) -> None: + return self._inner.remove_scorer(name)