diff --git a/CHANGELOG.md b/CHANGELOG.md index 295275f..122e817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to KonjoOS are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [unreleased] — Qdrant reset endpoint + +### Added +- `DELETE /ingest` (`api/main.py`) — wipes the Qdrant collection `/ingest` + writes to and returns `{points_removed}`. Scoped narrowly: the BM25 index, + semantic cache, and retrieval-by-id store are untouched. +- `QdrantStore.reset()` (`konjoai/store/qdrant.py`) — deletes and recreates + the collection, returning the point count it removed. +- Motivation: `/ingest` upserts under random UUID point ids, so nothing + deduplicates across repeated calls — a caller re-ingesting the same corpus + (e.g. an external eval harness) needs a way to start from a clean, + single-ingest index instead of accumulating points across runs. + +### Tests +- `api/test_api.py::test_reset_invokes_pipeline_reset_and_reports_points_removed` +- `tests/unit/test_auth.py::TestQdrantStoreTenantScoping::test_reset_deletes_and_recreates_the_collection` + ## [v1.7.0] — Sprint 27: Cache Warming + TTL Expiry + Query Clustering ### Added diff --git a/api/main.py b/api/main.py index 57e440d..50eee3e 100644 --- a/api/main.py +++ b/api/main.py @@ -10,6 +10,7 @@ GET /retrieval/{id} fetch a cached retrieval result by ID GET /metrics live performance stats POST /ingest index a batch of {text, metadata} documents + DELETE /ingest wipe the Qdrant collection /ingest writes to GET /health liveness check Singleflight: identical (tenant_id, question) requests collide on a single @@ -57,6 +58,10 @@ class IngestResult(BaseModel): sources: int +class ResetResult(BaseModel): + points_removed: int + + class QueryRequestBody(BaseModel): question: str = Field(..., min_length=1) tenant_id: str | None = Field(None, description="Tenant scope for this query.") @@ -105,6 +110,7 @@ class Pipeline(Protocol): async def embed_query(self, question: str) -> np.ndarray: ... async def retrieve(self, question: str, q_vec: np.ndarray, top_k: int) -> list[ScoredSource]: ... async def index(self, request: IngestRequest) -> IngestResult: ... + async def reset(self) -> int: ... def document_count(self) -> int: ... @@ -174,6 +180,12 @@ async def index(self, request: IngestRequest) -> IngestResult: return IngestResult(indexed=len(contents), sources=len(sources_seen)) + async def reset(self) -> int: + from konjoai.store.qdrant import get_store + + store = get_store() + return await asyncio.to_thread(store.reset) + def document_count(self) -> int: try: from konjoai.store.qdrant import get_store @@ -409,6 +421,22 @@ async def ingest( except (LookupError, ValueError): pass + # ── Reset ──────────────────────────────────────────────────────────────── + + @app.delete("/ingest", response_model=ResetResult, tags=["ingest"]) + async def reset_index(p: Pipeline = Depends(get_pipeline)) -> ResetResult: + """Wipe the Qdrant collection `/ingest` writes to. + + Scoped to the vector collection only — the BM25 index, semantic + cache, and retrieval-by-id store are untouched. Qdrant point ids are + random UUIDs (see `QdrantStore.upsert`), so nothing deduplicates + across repeated `/ingest` calls; a caller re-ingesting the same + corpus (e.g. an eval harness before an authoritative run) should + call this first to avoid an accumulating index. + """ + removed = await p.reset() + return ResetResult(points_removed=removed) + # ── Metrics ────────────────────────────────────────────────────────────── @app.get("/metrics", response_model=MetricsResult, tags=["health"]) diff --git a/api/test_api.py b/api/test_api.py index c53f7ef..0c3da01 100644 --- a/api/test_api.py +++ b/api/test_api.py @@ -35,6 +35,8 @@ class _StubPipeline: embed_calls: int = 0 retrieve_calls: int = 0 index_calls: int = 0 + reset_calls: int = 0 + reset_points_removed: int = 0 retrieve_delay_s: float = 0.0 embed_dim: int = 16 @@ -68,6 +70,10 @@ async def index(self, request: IngestRequest) -> IngestResult: sources=len({d.metadata.get("source") or "default" for d in request.documents}), ) + async def reset(self) -> int: + self.reset_calls += 1 + return self.reset_points_removed + def document_count(self) -> int: return self.document_count_value @@ -198,6 +204,16 @@ async def test_ingest_rejects_empty_payload(): assert r.status_code == 422 +@pytest.mark.asyncio +async def test_reset_invokes_pipeline_reset_and_reports_points_removed(): + pipeline = _StubPipeline(reset_points_removed=362) + async with _client(pipeline) as c: + r = await c.delete("/ingest") + assert r.status_code == 200 + assert r.json() == {"points_removed": 362} + assert pipeline.reset_calls == 1 + + # ── /metrics ─────────────────────────────────────────────────────────────── diff --git a/konjoai/store/qdrant.py b/konjoai/store/qdrant.py index aa3bf44..67d28de 100644 --- a/konjoai/store/qdrant.py +++ b/konjoai/store/qdrant.py @@ -156,6 +156,26 @@ def count(self) -> int: """Return the number of points in the collection.""" return self._client.count(collection_name=self._collection).count + def reset(self) -> int: + """Delete every point in the collection and recreate it empty. + + Point ids are random UUIDs (see :meth:`upsert`), so nothing + deduplicates across repeated ingests — this is what gives a caller + (e.g. an eval harness re-ingesting the same corpus) a clean, + single-ingest collection instead of an accumulating one. Returns the + number of points removed. + """ + from qdrant_client.models import Distance, VectorParams + + removed = self.count() + self._client.delete_collection(collection_name=self._collection) + self._client.create_collection( + collection_name=self._collection, + vectors_config=VectorParams(size=self._dim, distance=Distance.COSINE), + ) + logger.info("QdrantStore: reset collection '%s' (%d points removed)", self._collection, removed) + return removed + def scroll_all(self, batch_size: int = 256) -> tuple[np.ndarray, list[str], list[str], list[str]]: """Scroll through every point in the collection. diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index f23e68f..7c74187 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -425,3 +425,24 @@ def test_search_no_filter_when_tenant_unset(self) -> None: call_kwargs = mock_client.query_points.call_args.kwargs assert call_kwargs.get("query_filter") is None + + def test_reset_deletes_and_recreates_the_collection(self) -> None: + """reset() deletes+recreates the collection and returns the prior count.""" + from konjoai.store.qdrant import QdrantStore + + mock_client = MagicMock() + mock_client.count.return_value.count = 362 + + store = QdrantStore.__new__(QdrantStore) + store._client = mock_client + store._collection = "test_col" + store._dim = 4 + + removed = store.reset() + + assert removed == 362 + mock_client.delete_collection.assert_called_once_with(collection_name="test_col") + mock_client.create_collection.assert_called_once() + create_kwargs = mock_client.create_collection.call_args.kwargs + assert create_kwargs["collection_name"] == "test_col" + assert create_kwargs["vectors_config"].size == 4