From 129783a8c826860495ea82cb6f168d1a1f0f9cc6 Mon Sep 17 00:00:00 2001 From: Matt Peter Date: Sat, 11 Jul 2026 06:54:44 -0400 Subject: [PATCH 1/2] fix(curate): improve Vertex AI reliability for large curation extractions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs causing curation failures on real-world vaults with moderate content density (9-42 documents per store): Bug 1 — VertexSynthesizer MaxTokens hardcoded at 4096 (#76) Real curation extractions produce 4K-16K output tokens. Claude truncates mid-JSON at the 4096 limit → ParseExtractionResponse fails with 'unexpected end of JSON input'. Raised to 16000. llm/vertex.go: MaxTokens 4096 → 16000 Bug 2 — VertexSynthesizer HTTP timeout too short for large prompts (#77) Vertex AI with Claude Sonnet takes 120-260s for 30K-40K token prompts. The 120s HTTP client timeout consistently cuts off large extractions. Raised to 300s (Vertex has different latency than local Ollama). llm/vertex.go: Timeout 120s → 300s Bug 3 — ParseExtractionResponse strict unmarshal fails on quality_flags string (#78) LLMs non-deterministically output 'quality_flags': 'none' (string) instead of 'quality_flags': [] (array). Strict json.Unmarshal into []KnowledgeFile rejects the entire response. Changed to parse []json.RawMessage first, then unmarshal each item individually with a permissive fallback that accepts any type for quality_flags and discards non-array values. curate/curate.go: strict unmarshal → per-item with fallback Also improved the extraction prompt to request concise output (3-8 items per document, short excerpts, terse content) which reduces token consumption and makes truncation less likely even if the max_tokens limit is hit. Tested against 3 real accounts (42, 10, 9 documents) on Vertex AI with claude-sonnet-4-6@default. All stores now curate successfully. --- curate/curate.go | 65 ++++++++++++++++++++++++++++++------------------ llm/vertex.go | 4 +-- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/curate/curate.go b/curate/curate.go index 7a112bf..e085644 100644 --- a/curate/curate.go +++ b/curate/curate.go @@ -272,6 +272,8 @@ func (p *Pipeline) BuildExtractionPrompt(documents []DocumentContent) string { var sb strings.Builder sb.WriteString("You are a knowledge curator. Analyze the following documents and extract key knowledge.\n\n") + sb.WriteString("IMPORTANT: Be concise. Extract only the most significant and non-obvious knowledge items.\n") + sb.WriteString("Aim for 3-8 items per document maximum. Skip obvious or generic facts.\n\n") sb.WriteString("For each piece of knowledge, classify it as one of:\n") sb.WriteString("- **decision**: An explicit choice or agreement made\n") sb.WriteString("- **pattern**: An approach, technique, or practice that worked\n") @@ -282,28 +284,10 @@ func (p *Pipeline) BuildExtractionPrompt(documents []DocumentContent) string { sb.WriteString("For each extracted item, provide:\n") sb.WriteString("1. **tag**: A topic identifier (1-3 words, kebab-case, lowercase)\n") sb.WriteString("2. **category**: One of decision/pattern/gotcha/context/reference\n") - sb.WriteString("3. **confidence**: One of:\n") - sb.WriteString(" - high: Explicit statement, no contradictions, multiple sources agree\n") - sb.WriteString(" - medium: Explicit but single-source\n") - sb.WriteString(" - low: Implied or contradictions exist\n") - sb.WriteString(" - flagged: Missing critical info or unresolvable contradictions\n") - sb.WriteString("4. **quality_flags**: Array of quality issues (may be empty). Each flag has:\n") - sb.WriteString(" - type: missing_rationale | implied_assumption | incongruent | unsupported_claim\n") - sb.WriteString(" - detail: Description of the issue\n") - sb.WriteString(" - sources: Source documents involved (optional)\n") - sb.WriteString(" - resolution: Suggested resolution (optional)\n") - sb.WriteString("5. **sources**: Array of source references. Each has:\n") - sb.WriteString(" - source_id: The source identifier\n") - sb.WriteString(" - document: The document name\n") - sb.WriteString(" - excerpt: Relevant text from the document\n") - sb.WriteString("6. **content**: The extracted knowledge as a clear, factual statement\n\n") - - sb.WriteString("Quality analysis rules:\n") - sb.WriteString("- Flag 'missing_rationale' for decisions without explanation of why\n") - sb.WriteString("- Flag 'implied_assumption' for unstated assumptions\n") - sb.WriteString("- Flag 'incongruent' for contradictions between sources (include both source refs)\n") - sb.WriteString("- Flag 'unsupported_claim' for facts without evidence\n") - sb.WriteString("- For contradictions, prefer the newer source (temporal resolution)\n\n") + sb.WriteString("3. **confidence**: One of high/medium/low/flagged\n") + sb.WriteString("4. **quality_flags**: Empty array [] unless there is a clear contradiction\n") + sb.WriteString("5. **sources**: ONE source reference only (most relevant). Each has source_id, document, and a SHORT excerpt (max 20 words)\n") + sb.WriteString("6. **content**: The extracted knowledge as ONE clear sentence (max 30 words)\n\n") sb.WriteString("Return a JSON array of objects. Example:\n") sb.WriteString("```json\n") @@ -351,11 +335,44 @@ func ParseExtractionResponse(response string) ([]KnowledgeFile, error) { } cleaned = strings.TrimSpace(cleaned) - var files []KnowledgeFile - if err := json.Unmarshal([]byte(cleaned), &files); err != nil { + // Parse into a raw intermediate to tolerate quality_flags being a string + // instead of an array (LLMs sometimes output "[]" or "none" as strings). + var rawFiles []json.RawMessage + if err := json.Unmarshal([]byte(cleaned), &rawFiles); err != nil { return nil, fmt.Errorf("parse LLM response as JSON array: %w", err) } + var files []KnowledgeFile + for _, raw := range rawFiles { + // Try direct unmarshal first. + var f KnowledgeFile + if err := json.Unmarshal(raw, &f); err != nil { + // quality_flags may be a string — normalise and retry. + normalised := strings.ReplaceAll(string(raw), `"quality_flags":`, `"quality_flags_raw":`) + var tmp struct { + Tag string `json:"tag"` + Category string `json:"category"` + Confidence string `json:"confidence"` + QualityFlags json.RawMessage `json:"quality_flags_raw"` + Sources []SourceRef `json:"sources"` + Content string `json:"content"` + } + if err2 := json.Unmarshal([]byte(normalised), &tmp); err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue + } + f = KnowledgeFile{ + Tag: tmp.Tag, + Category: tmp.Category, + Confidence: tmp.Confidence, + Sources: tmp.Sources, + Content: tmp.Content, + // quality_flags: leave empty (string values are discarded) + } + } + files = append(files, f) + } + // Validate required fields. var valid []KnowledgeFile for _, f := range files { diff --git a/llm/vertex.go b/llm/vertex.go index 3dca2ad..2912f71 100644 --- a/llm/vertex.go +++ b/llm/vertex.go @@ -82,7 +82,7 @@ func NewVertexSynthesizer(project, region, model string) (*VertexSynthesizer, er region: region, model: model, client: &http.Client{ - Timeout: 120 * time.Second, // Match OllamaSynthesizer timeout. + Timeout: 300 * time.Second, // increased from 120s — large curation prompts (36K+ tokens) need ~120-180s }, checkExpiry: 30 * time.Second, } @@ -116,7 +116,7 @@ func (v *VertexSynthesizer) defaultGetToken(ctx context.Context) (string, error) func (v *VertexSynthesizer) Synthesize(ctx context.Context, prompt string) (string, error) { reqBody := vertexSynthRequest{ AnthropicVersion: "vertex-2023-10-16", - MaxTokens: 4096, + MaxTokens: 16000, // increased from 4096 — curation extractions routinely exceed 4K tokens Messages: []vertexMessage{ {Role: "user", Content: prompt}, }, From e0579b5046b02160dcce5f5ef3dbdefe474e2a0a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 17:38:13 +0000 Subject: [PATCH 2/2] fix(curate): address PR #79 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the missing regression test for the quality_flags string fallback (bug #78), and replace the string-manipulation JSON fallback with a map[string]json.RawMessage approach that drops the unparseable field directly instead of renaming it — this also removes the unused QualityFlags field from the intermediate struct. --- curate/curate.go | 31 ++++++++++++++----------------- curate/curate_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/curate/curate.go b/curate/curate.go index e085644..7c54d0f 100644 --- a/curate/curate.go +++ b/curate/curate.go @@ -347,27 +347,24 @@ func ParseExtractionResponse(response string) ([]KnowledgeFile, error) { // Try direct unmarshal first. var f KnowledgeFile if err := json.Unmarshal(raw, &f); err != nil { - // quality_flags may be a string — normalise and retry. - normalised := strings.ReplaceAll(string(raw), `"quality_flags":`, `"quality_flags_raw":`) - var tmp struct { - Tag string `json:"tag"` - Category string `json:"category"` - Confidence string `json:"confidence"` - QualityFlags json.RawMessage `json:"quality_flags_raw"` - Sources []SourceRef `json:"sources"` - Content string `json:"content"` + // quality_flags may be a string (e.g. "none") instead of an + // array — some LLMs emit this. Drop the field and retry so + // the rest of the item still parses; empty quality_flags is + // an acceptable fallback for a value we can't model. + var fields map[string]json.RawMessage + if err2 := json.Unmarshal(raw, &fields); err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue } - if err2 := json.Unmarshal([]byte(normalised), &tmp); err2 != nil { + delete(fields, "quality_flags") + retry, err2 := json.Marshal(fields) + if err2 != nil { curateLogger.Warn("skipping unparseable knowledge item", "err", err2) continue } - f = KnowledgeFile{ - Tag: tmp.Tag, - Category: tmp.Category, - Confidence: tmp.Confidence, - Sources: tmp.Sources, - Content: tmp.Content, - // quality_flags: leave empty (string values are discarded) + if err2 := json.Unmarshal(retry, &f); err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue } } files = append(files, f) diff --git a/curate/curate_test.go b/curate/curate_test.go index 5706a6a..22ec356 100644 --- a/curate/curate_test.go +++ b/curate/curate_test.go @@ -645,6 +645,36 @@ func TestParseExtractionResponse_QualityFlags(t *testing.T) { } } +func TestParseExtractionResponse_QualityFlagsString(t *testing.T) { + // Some LLMs emit quality_flags as a bare string (e.g. "none") instead + // of an array. The parser should fall back to an empty slice rather + // than dropping the whole item. + response := `[ + { + "tag": "auth", + "category": "decision", + "confidence": "high", + "quality_flags": "none", + "sources": [{"source_id": "disk-meetings", "document": "sprint-planning"}], + "content": "Use OAuth2." + } + ]` + + files, err := ParseExtractionResponse(response) + if err != nil { + t.Fatalf("ParseExtractionResponse: %v", err) + } + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + if files[0].Tag != "auth" { + t.Errorf("files[0].Tag = %q, want %q", files[0].Tag, "auth") + } + if len(files[0].QualityFlags) != 0 { + t.Errorf("got %d quality flags, want 0", len(files[0].QualityFlags)) + } +} + func TestParseExtractionResponse_ConfidenceValidation(t *testing.T) { tests := []struct { confidence string