fix(recall): collapse duplicate raw facts before truncation

.env.example

@@ -80,6 +80,10 @@ HINDSIGHT_API_LOG_LEVEL=info
 # Unset uses HINDSIGHT_API_RETAIN_CHUNK_SIZE as the structured-chunk limit.
 # HINDSIGHT_API_RETAIN_STRUCTURED_CHUNK_SIZE=
 
+# Recall duplicate filtering. 1.0 collapses exact normalized duplicate raw facts
+# after ranking and before token filtering; lower values are fuzzier. 0 disables.
+# HINDSIGHT_API_RECALL_DEDUP_THRESHOLD=1.0
+
 # Dry-run extraction preview endpoint (POST /memories/dry-run-extract). Enabled by default; it makes
 # a real LLM call but stores nothing. Set to false to remove the endpoint (returns 404).
 # HINDSIGHT_API_ENABLE_DRY_RUN_EXTRACT=true

hindsight-api-slim/hindsight_api/config.py

@@ -624,6 +624,7 @@ def _resolve_operation_temperature(operation_env: str, default: float) -> float
 ENV_RECALL_INCLUDE_CHUNKS = "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
 ENV_RECALL_MAX_TOKENS = "HINDSIGHT_API_RECALL_MAX_TOKENS"
 ENV_RECALL_CHUNKS_MAX_TOKENS = "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+ENV_RECALL_DEDUP_THRESHOLD = "HINDSIGHT_API_RECALL_DEDUP_THRESHOLD"
 
 # Recall budget mapping (budget enum -> thinking_budget integer)
 ENV_RECALL_BUDGET_FUNCTION = "HINDSIGHT_API_RECALL_BUDGET_FUNCTION"
@@ -1060,6 +1061,7 @@ def _parse_strategy_boosts(raw: str | None) -> dict[str, str]:
 DEFAULT_RECALL_INCLUDE_CHUNKS = True  # Whether internal recall (e.g. mental model refresh) returns raw chunks
 DEFAULT_RECALL_MAX_TOKENS = 2048  # Token budget for facts returned by internal recall
 DEFAULT_RECALL_CHUNKS_MAX_TOKENS = 1000  # Token budget for raw chunks returned by internal recall
+DEFAULT_RECALL_DEDUP_THRESHOLD = 1.0  # 1.0 collapses exact normalized raw-fact duplicates; 0 disables
 
 # Recall budget mapping
 # "fixed": thinking_budget = recall_budget_fixed_<level> (preserves legacy behavior)
@@ -1863,6 +1865,7 @@ class HindsightConfig:
     recall_include_chunks: bool
     recall_max_tokens: int
     recall_chunks_max_tokens: int
+    recall_dedup_threshold: float
 
     # Recall budget mapping: how the Budget enum (LOW/MID/HIGH) maps to thinking_budget integer.
     # function="fixed": use the recall_budget_fixed_* values directly (legacy behavior).
@@ -2066,6 +2069,7 @@ class HindsightConfig:
         "recall_include_chunks",
         "recall_max_tokens",
         "recall_chunks_max_tokens",
+        "recall_dedup_threshold",
         # Recall budget mapping (Budget enum -> thinking_budget integer)
         "recall_budget_function",
         "recall_budget_fixed_low",
@@ -2179,6 +2183,11 @@ def validate(self) -> None:
                 f"Invalid semantic_min_similarity: {self.semantic_min_similarity}. Must be between 0.0 and 1.0"
             )
 
+        if not 0.0 <= self.recall_dedup_threshold <= 1.0:
+            raise ValueError(
+                f"Invalid recall_dedup_threshold: {self.recall_dedup_threshold}. Must be between 0.0 and 1.0"
+            )
+
         # Validate bedrock_service_tier
         valid_bedrock_tiers = (None, "flex", "priority", "reserved")
         if self.llm_bedrock_service_tier not in valid_bedrock_tiers:
@@ -2941,6 +2950,7 @@ def from_env(cls) -> "HindsightConfig":
             recall_chunks_max_tokens=int(
                 os.getenv(ENV_RECALL_CHUNKS_MAX_TOKENS, str(DEFAULT_RECALL_CHUNKS_MAX_TOKENS))
             ),
+            recall_dedup_threshold=float(os.getenv(ENV_RECALL_DEDUP_THRESHOLD, str(DEFAULT_RECALL_DEDUP_THRESHOLD))),
             recall_budget_function=_validate_recall_budget_function(
                 os.getenv(ENV_RECALL_BUDGET_FUNCTION, DEFAULT_RECALL_BUDGET_FUNCTION)
             ),

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -15,6 +15,7 @@
 import inspect
 import json
 import logging
+import math
 import sys
 import time
 import uuid
@@ -30,6 +31,7 @@
 from ..cancellation import OperationCancelledError
 from ..config import (
     DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
+    DEFAULT_RECALL_DEDUP_THRESHOLD,
     DEFAULT_RECALL_INCLUDE_CHUNKS,
     DEFAULT_RECALL_MAX_TOKENS,
     DEFAULT_REFLECT_SOURCE_FACTS_MAX_TOKENS,
@@ -760,6 +762,29 @@ def _recall_scoring_now(question_date: datetime | None) -> datetime:
 from .db_utils import acquire_with_retry
 
 
+def _resolve_recall_dedup_threshold(raw_value: Any) -> float:
+    """Coerce bank-configured recall dedup thresholds without breaking recall."""
+    if raw_value is None:
+        return DEFAULT_RECALL_DEDUP_THRESHOLD
+    try:
+        threshold = float(raw_value)
+    except (TypeError, ValueError):
+        logger.warning(
+            "Invalid recall_dedup_threshold=%r; using default %.3f",
+            raw_value,
+            DEFAULT_RECALL_DEDUP_THRESHOLD,
+        )
+        return DEFAULT_RECALL_DEDUP_THRESHOLD
+    if not math.isfinite(threshold):
+        logger.warning(
+            "Non-finite recall_dedup_threshold=%r; using default %.3f",
+            raw_value,
+            DEFAULT_RECALL_DEDUP_THRESHOLD,
+        )
+        return DEFAULT_RECALL_DEDUP_THRESHOLD
+    return max(0.0, min(1.0, threshold))
+
+
 def _get_tiktoken_encoding():
     """Get cached tiktoken encoding (cl100k_base for GPT-4/3.5).
 
@@ -4147,6 +4172,9 @@ async def recall_async(
         # derives from max_tokens and clamps to [recall_budget_min, recall_budget_max].
         budget_config_dict = await self._config_resolver.get_bank_config(bank_id, request_context)
         thinking_budget = _resolve_thinking_budget(budget_config_dict, budget, max_tokens)
+        recall_dedup_threshold = _resolve_recall_dedup_threshold(
+            budget_config_dict.get("recall_dedup_threshold", DEFAULT_RECALL_DEDUP_THRESHOLD)
+        )
 
         # Log recall start with tags if present (skip if quiet mode for internal operations)
         if not _quiet:
@@ -4204,6 +4232,7 @@ async def recall_async(
                             max_source_facts_tokens=max_source_facts_tokens,
                             max_source_facts_tokens_per_observation=max_source_facts_tokens_per_observation,
                             reranking=reranking,
+                            recall_dedup_threshold=recall_dedup_threshold,
                         )
                         break  # Success - exit retry loop
                     except OperationCancelledError:
@@ -4342,6 +4371,7 @@ async def _search_with_retries(
         max_source_facts_tokens: int = 4096,
         max_source_facts_tokens_per_observation: int = -1,
         reranking: RecallReranking = "cross_encoder",
+        recall_dedup_threshold: float = DEFAULT_RECALL_DEDUP_THRESHOLD,
     ) -> RecallResultModel:
         """
         Search implementation with modular retrieval and reranking.
@@ -4940,6 +4970,25 @@ def to_tuple_format(results):
                             f"raw fact(s) superseded by {len(observation_ids)} observation(s)"
                         )
 
+            if recall_dedup_threshold > 0.0 and scored_results:
+                dedup_start = time.time()
+                from .search.dedup import collapse_near_duplicate_raw_facts
+
+                before_dedup = len(scored_results)
+                scored_results = collapse_near_duplicate_raw_facts(scored_results, recall_dedup_threshold)
+                dropped_dedup = before_dedup - len(scored_results)
+                if dropped_dedup:
+                    log_buffer.append(
+                        f"  [4.10] recall_dedup(threshold={recall_dedup_threshold:.3f}): "
+                        f"dropped {dropped_dedup} near-duplicate raw fact(s)"
+                    )
+                if tracer:
+                    tracer.add_phase_metric(
+                        "recall_dedup",
+                        time.time() - dedup_start,
+                        {"threshold": recall_dedup_threshold, "dropped": dropped_dedup},
+                    )
+
             # Step 5: Truncate to thinking_budget * 2 for token filtering
             rerank_limit = thinking_budget * 2
             top_scored = scored_results[:rerank_limit]

hindsight-api-slim/hindsight_api/engine/search/dedup.py

@@ -0,0 +1,79 @@
+"""Recall result deduplication helpers."""
+
+import re
+from difflib import SequenceMatcher
+
+from .types import ScoredResult
+
+_RAW_FACT_TYPES = frozenset({"world", "experience"})
+_WORD_RE = re.compile(r"\w+")
+
+
+def normalize_recall_dedup_text(text: str) -> str:
+    """Normalize recall text for duplicate comparison."""
+    return " ".join(_WORD_RE.findall(text.casefold()))
+
+
+def recall_text_similarity(left: str, right: str) -> float:
+    """Return normalized lexical similarity for two recall result texts."""
+    left_normalized = normalize_recall_dedup_text(left)
+    right_normalized = normalize_recall_dedup_text(right)
+    if not left_normalized or not right_normalized:
+        return 0.0
+    if left_normalized == right_normalized:
+        return 1.0
+    return SequenceMatcher(None, left_normalized, right_normalized).ratio()
+
+
+def collapse_near_duplicate_raw_facts(
+    scored_results: list[ScoredResult],
+    threshold: float,
+) -> list[ScoredResult]:
+    """Drop lower-ranked raw facts whose text duplicates an earlier raw fact.
+
+    The input is already sorted by final recall rank. Dedup therefore keeps the
+    first/highest-ranked result in each duplicate cluster and lets downstream
+    truncation backfill freed slots from later candidates.
+    """
+    if threshold <= 0.0:
+        return list(scored_results)
+
+    retained_results: list[ScoredResult] = []
+    retained_world_texts: list[str] = []
+    retained_experience_texts: list[str] = []
+    retained_world_text_set: set[str] = set()
+    retained_experience_text_set: set[str] = set()
+
+    for scored_result in scored_results:
+        retrieval = scored_result.retrieval
+        if retrieval.fact_type not in _RAW_FACT_TYPES:
+            retained_results.append(scored_result)
+            continue
+
+        normalized_text = normalize_recall_dedup_text(retrieval.text)
+        if not normalized_text:
+            retained_results.append(scored_result)
+            continue
+
+        retained_texts = retained_world_texts
+        retained_text_set = retained_world_text_set
+        if retrieval.fact_type == "experience":
+            retained_texts = retained_experience_texts
+            retained_text_set = retained_experience_text_set
+
+        if threshold >= 1.0:
+            is_duplicate = normalized_text in retained_text_set
+        else:
+            is_duplicate = any(
+                retained_text == normalized_text
+                or SequenceMatcher(None, retained_text, normalized_text).ratio() >= threshold
+                for retained_text in retained_texts
+            )
+        if is_duplicate:
+            continue
+
+        retained_results.append(scored_result)
+        retained_texts.append(normalized_text)
+        retained_text_set.add(normalized_text)
+
+    return retained_results

hindsight-api-slim/tests/test_recall_config.py

@@ -1,6 +1,5 @@
 """
-Tests for the internal recall configuration knobs used during mental model
-refresh: recall_include_chunks, recall_max_tokens, recall_chunks_max_tokens.
+Tests for recall configuration knobs.
 
 These are exposed both as hierarchical config fields (env → tenant → bank)
 and as overrides on a mental model's `trigger` JSONB field.
@@ -71,6 +70,7 @@ def test_fields_exist_on_dataclass(self):
         assert "recall_include_chunks" in names
         assert "recall_max_tokens" in names
         assert "recall_chunks_max_tokens" in names
+        assert "recall_dedup_threshold" in names
 
     def test_fields_are_configurable(self):
         from hindsight_api.config import HindsightConfig
@@ -79,35 +79,41 @@ def test_fields_are_configurable(self):
         assert "recall_include_chunks" in configurable
         assert "recall_max_tokens" in configurable
         assert "recall_chunks_max_tokens" in configurable
+        assert "recall_dedup_threshold" in configurable
 
     def test_default_values(self):
         from hindsight_api.config import (
             DEFAULT_RECALL_CHUNKS_MAX_TOKENS,
+            DEFAULT_RECALL_DEDUP_THRESHOLD,
             DEFAULT_RECALL_INCLUDE_CHUNKS,
             DEFAULT_RECALL_MAX_TOKENS,
         )
 
         assert DEFAULT_RECALL_INCLUDE_CHUNKS is True
         assert DEFAULT_RECALL_MAX_TOKENS == 2048
         assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000
+        assert DEFAULT_RECALL_DEDUP_THRESHOLD == 1.0
 
     def test_env_var_constants(self):
         from hindsight_api.config import (
             ENV_RECALL_CHUNKS_MAX_TOKENS,
+            ENV_RECALL_DEDUP_THRESHOLD,
             ENV_RECALL_INCLUDE_CHUNKS,
             ENV_RECALL_MAX_TOKENS,
         )
 
         assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS"
         assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS"
         assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS"
+        assert ENV_RECALL_DEDUP_THRESHOLD == "HINDSIGHT_API_RECALL_DEDUP_THRESHOLD"
 
     @patch.dict(
         "os.environ",
         {
             "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false",
             "HINDSIGHT_API_RECALL_MAX_TOKENS": "777",
             "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333",
+            "HINDSIGHT_API_RECALL_DEDUP_THRESHOLD": "0.96",
         },
     )
     def test_from_env_reads_overrides(self):
@@ -117,6 +123,14 @@ def test_from_env_reads_overrides(self):
         assert config.recall_include_chunks is False
         assert config.recall_max_tokens == 777
         assert config.recall_chunks_max_tokens == 333
+        assert config.recall_dedup_threshold == 0.96
+
+    @patch.dict("os.environ", {"HINDSIGHT_API_RECALL_DEDUP_THRESHOLD": "1.1"})
+    def test_dedup_threshold_rejects_out_of_range_env(self):
+        from hindsight_api.config import HindsightConfig
+
+        with pytest.raises(ValueError, match="recall_dedup_threshold"):
+            HindsightConfig.from_env()
 
 
 class TestMentalModelTriggerRecallFields: