diff --git a/curate/curate.go b/curate/curate.go index 7a112bf..7c54d0f 100644 --- a/curate/curate.go +++ b/curate/curate.go @@ -272,6 +272,8 @@ func (p *Pipeline) BuildExtractionPrompt(documents []DocumentContent) string { var sb strings.Builder sb.WriteString("You are a knowledge curator. Analyze the following documents and extract key knowledge.\n\n") + sb.WriteString("IMPORTANT: Be concise. Extract only the most significant and non-obvious knowledge items.\n") + sb.WriteString("Aim for 3-8 items per document maximum. Skip obvious or generic facts.\n\n") sb.WriteString("For each piece of knowledge, classify it as one of:\n") sb.WriteString("- **decision**: An explicit choice or agreement made\n") sb.WriteString("- **pattern**: An approach, technique, or practice that worked\n") @@ -282,28 +284,10 @@ func (p *Pipeline) BuildExtractionPrompt(documents []DocumentContent) string { sb.WriteString("For each extracted item, provide:\n") sb.WriteString("1. **tag**: A topic identifier (1-3 words, kebab-case, lowercase)\n") sb.WriteString("2. **category**: One of decision/pattern/gotcha/context/reference\n") - sb.WriteString("3. **confidence**: One of:\n") - sb.WriteString(" - high: Explicit statement, no contradictions, multiple sources agree\n") - sb.WriteString(" - medium: Explicit but single-source\n") - sb.WriteString(" - low: Implied or contradictions exist\n") - sb.WriteString(" - flagged: Missing critical info or unresolvable contradictions\n") - sb.WriteString("4. **quality_flags**: Array of quality issues (may be empty). Each flag has:\n") - sb.WriteString(" - type: missing_rationale | implied_assumption | incongruent | unsupported_claim\n") - sb.WriteString(" - detail: Description of the issue\n") - sb.WriteString(" - sources: Source documents involved (optional)\n") - sb.WriteString(" - resolution: Suggested resolution (optional)\n") - sb.WriteString("5. **sources**: Array of source references. Each has:\n") - sb.WriteString(" - source_id: The source identifier\n") - sb.WriteString(" - document: The document name\n") - sb.WriteString(" - excerpt: Relevant text from the document\n") - sb.WriteString("6. **content**: The extracted knowledge as a clear, factual statement\n\n") - - sb.WriteString("Quality analysis rules:\n") - sb.WriteString("- Flag 'missing_rationale' for decisions without explanation of why\n") - sb.WriteString("- Flag 'implied_assumption' for unstated assumptions\n") - sb.WriteString("- Flag 'incongruent' for contradictions between sources (include both source refs)\n") - sb.WriteString("- Flag 'unsupported_claim' for facts without evidence\n") - sb.WriteString("- For contradictions, prefer the newer source (temporal resolution)\n\n") + sb.WriteString("3. **confidence**: One of high/medium/low/flagged\n") + sb.WriteString("4. **quality_flags**: Empty array [] unless there is a clear contradiction\n") + sb.WriteString("5. **sources**: ONE source reference only (most relevant). Each has source_id, document, and a SHORT excerpt (max 20 words)\n") + sb.WriteString("6. **content**: The extracted knowledge as ONE clear sentence (max 30 words)\n\n") sb.WriteString("Return a JSON array of objects. Example:\n") sb.WriteString("```json\n") @@ -351,11 +335,41 @@ func ParseExtractionResponse(response string) ([]KnowledgeFile, error) { } cleaned = strings.TrimSpace(cleaned) - var files []KnowledgeFile - if err := json.Unmarshal([]byte(cleaned), &files); err != nil { + // Parse into a raw intermediate to tolerate quality_flags being a string + // instead of an array (LLMs sometimes output "[]" or "none" as strings). + var rawFiles []json.RawMessage + if err := json.Unmarshal([]byte(cleaned), &rawFiles); err != nil { return nil, fmt.Errorf("parse LLM response as JSON array: %w", err) } + var files []KnowledgeFile + for _, raw := range rawFiles { + // Try direct unmarshal first. + var f KnowledgeFile + if err := json.Unmarshal(raw, &f); err != nil { + // quality_flags may be a string (e.g. "none") instead of an + // array — some LLMs emit this. Drop the field and retry so + // the rest of the item still parses; empty quality_flags is + // an acceptable fallback for a value we can't model. + var fields map[string]json.RawMessage + if err2 := json.Unmarshal(raw, &fields); err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue + } + delete(fields, "quality_flags") + retry, err2 := json.Marshal(fields) + if err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue + } + if err2 := json.Unmarshal(retry, &f); err2 != nil { + curateLogger.Warn("skipping unparseable knowledge item", "err", err2) + continue + } + } + files = append(files, f) + } + // Validate required fields. var valid []KnowledgeFile for _, f := range files { diff --git a/curate/curate_test.go b/curate/curate_test.go index 5706a6a..22ec356 100644 --- a/curate/curate_test.go +++ b/curate/curate_test.go @@ -645,6 +645,36 @@ func TestParseExtractionResponse_QualityFlags(t *testing.T) { } } +func TestParseExtractionResponse_QualityFlagsString(t *testing.T) { + // Some LLMs emit quality_flags as a bare string (e.g. "none") instead + // of an array. The parser should fall back to an empty slice rather + // than dropping the whole item. + response := `[ + { + "tag": "auth", + "category": "decision", + "confidence": "high", + "quality_flags": "none", + "sources": [{"source_id": "disk-meetings", "document": "sprint-planning"}], + "content": "Use OAuth2." + } + ]` + + files, err := ParseExtractionResponse(response) + if err != nil { + t.Fatalf("ParseExtractionResponse: %v", err) + } + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + if files[0].Tag != "auth" { + t.Errorf("files[0].Tag = %q, want %q", files[0].Tag, "auth") + } + if len(files[0].QualityFlags) != 0 { + t.Errorf("got %d quality flags, want 0", len(files[0].QualityFlags)) + } +} + func TestParseExtractionResponse_ConfidenceValidation(t *testing.T) { tests := []struct { confidence string diff --git a/llm/vertex.go b/llm/vertex.go index 3dca2ad..2912f71 100644 --- a/llm/vertex.go +++ b/llm/vertex.go @@ -82,7 +82,7 @@ func NewVertexSynthesizer(project, region, model string) (*VertexSynthesizer, er region: region, model: model, client: &http.Client{ - Timeout: 120 * time.Second, // Match OllamaSynthesizer timeout. + Timeout: 300 * time.Second, // increased from 120s — large curation prompts (36K+ tokens) need ~120-180s }, checkExpiry: 30 * time.Second, } @@ -116,7 +116,7 @@ func (v *VertexSynthesizer) defaultGetToken(ctx context.Context) (string, error) func (v *VertexSynthesizer) Synthesize(ctx context.Context, prompt string) (string, error) { reqBody := vertexSynthRequest{ AnthropicVersion: "vertex-2023-10-16", - MaxTokens: 4096, + MaxTokens: 16000, // increased from 4096 — curation extractions routinely exceed 4K tokens Messages: []vertexMessage{ {Role: "user", Content: prompt}, },