From c8b95c0b58c5892042e398aa17a23ec80f9aa234 Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:25:37 -0500 Subject: [PATCH 01/10] feat(setup): add provider defaults and lane status Configure unset retrieval and people lanes from available providers with explicit consent and recommended defaults. Report which lanes are active and the steps needed to finish setup. - feat(setup): keep provider onboarding output consistent with the pass - fix(setup): keep the local fallback local and disclose every source Refs #634 Co-authored-by: Claude Fable 5.1 Generated with Codex --- cmd/msgvault/cmd/setup.go | 8 +- cmd/msgvault/cmd/setup_lanes.go | 517 ++++++++++ cmd/msgvault/cmd/setup_providers.go | 1092 ++++++++++++++++++++++ cmd/msgvault/cmd/setup_providers_test.go | 584 ++++++++++++ cmd/msgvault/cmd/setup_status.go | 56 ++ docs/changelog.md | 15 + docs/cli-reference.md | 45 + docs/configuration.md | 64 ++ docs/setup.md | 17 + docs/usage/recommended-configuration.md | 211 +++++ docs/zensical.toml | 1 + 11 files changed, 2609 insertions(+), 1 deletion(-) create mode 100644 cmd/msgvault/cmd/setup_lanes.go create mode 100644 cmd/msgvault/cmd/setup_providers.go create mode 100644 cmd/msgvault/cmd/setup_providers_test.go create mode 100644 cmd/msgvault/cmd/setup_status.go create mode 100644 docs/usage/recommended-configuration.md diff --git a/cmd/msgvault/cmd/setup.go b/cmd/msgvault/cmd/setup.go index 78f1f52e8..de7a6e47a 100644 --- a/cmd/msgvault/cmd/setup.go +++ b/cmd/msgvault/cmd/setup.go @@ -26,12 +26,18 @@ This command helps you: 2. Create the config.toml file 3. Optionally configure a remote NAS server for token export -Run this once after installing msgvault to get started quickly.`, +Run this once after installing msgvault to get started quickly. Then run +"msgvault setup providers" to turn on search, attachment, and people lanes +from the API keys you have, and "msgvault setup status" to see what is on.`, Args: cobra.NoArgs, RunE: runSetup, } func init() { + setupCmd.AddCommand( + newSetupProvidersCommand(defaultSetupProvidersDeps()), + newSetupStatusCommand(defaultSetupStatusDeps()), + ) rootCmd.AddCommand(setupCmd) } diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go new file mode 100644 index 000000000..ae5828869 --- /dev/null +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -0,0 +1,517 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "text/tabwriter" + + "go.kenn.io/msgvault/internal/attachmentpolicy" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" +) + +// Lane and consent states shared by `setup providers` and `setup status`. +const ( + laneStateOn = "on" + laneStateOff = "off" + laneStatePending = "pending" + + consentActive = "active" + consentMissing = "missing" + consentUnknown = "unknown" + + laneTextSearch = "text_search" + lanePersonSearch = "person_search" + laneVisualSearch = "visual_search" + laneDocuments = "documents" + laneDocumentVectors = "document_vectors" + lanePeopleInference = "people_inference" + laneActivity = "activity" + laneMediaPolicy = "media_policy" + + // Recommended provider defaults. These are the values setup writes when + // nothing is configured; every one of them remains settable per lane. + setupVoyageKeyEnv = "VOYAGE_API_KEY" // #nosec G101 -- environment variable name, not a credential. + setupOpenAIKeyEnv = "OPENAI_API_KEY" // #nosec G101 -- environment variable name, not a credential. + setupVoyageEndpoint = "https://api.voyageai.com/v1" + setupVoyageTextModel = "voyage-context-4" + setupVoyageTextDim = 1024 + setupOpenAIEndpoint = "https://api.openai.com/v1" + setupOpenAITextModel = "text-embedding-3-small" + setupOpenAITextDim = 1536 + setupOllamaTextModel = "nomic-embed-text" + setupOllamaTextDim = 768 + setupOllamaDocPrefix = "search_document: " + setupOllamaQueryPrefix = "search_query: " + setupOllamaMaxInput = 2000 + setupEmbedCron = "*/15 * * * *" + setupInferenceModel = "gpt-5.6-luna" + setupInferenceReasoning = "medium" + setupInferenceProfile = "openai" + setupOllamaProfile = "ollama" + setupPostureDeclared = "provider-declared" + + setupVoyageManifestName = "voyage-capabilities.json" + setupMistralManifestName = "mistral-capabilities.json" +) + +// laneStatus is one row of the provider report. It answers, for one lane: +// which provider and model, whether it is on, why not, and what to run next. +type laneStatus struct { + Lane string `json:"lane"` + Label string `json:"label"` + State string `json:"state"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + Schedule string `json:"schedule,omitempty"` + Consent string `json:"consent,omitempty"` + Reason string `json:"reason,omitempty"` + Next []string `json:"next,omitempty"` +} + +// laneReport is the complete report printed by both setup subcommands. +type laneReport struct { + ConfigPath string `json:"config_path"` + Lanes []laneStatus `json:"lanes"` + MCPTools []string `json:"mcp_tools_live"` +} + +// setupConsentState is a best-effort view of recorded consents. A nil +// pointer means the archive could not be read (missing database, daemon +// incompatibility, PostgreSQL unreachable) and every consent is reported as +// unknown rather than missing. +type setupConsentState struct { + Documents bool + Visual bool + PersonInference bool + PersonSemantic bool +} + +// setupEnvironment is what the report needs beyond the loaded config: the +// process environment, the filesystem, and the archive's consent records. +type setupEnvironment struct { + lookupEnv func(string) (string, bool) + fileExists func(string) bool + consent *setupConsentState +} + +func (e setupEnvironment) hasEnv(name string) bool { + if e.lookupEnv == nil || name == "" { + return false + } + value, ok := e.lookupEnv(name) + return ok && strings.TrimSpace(value) != "" +} + +func (e setupEnvironment) exists(path string) bool { + if e.fileExists == nil || path == "" { + return false + } + return e.fileExists(path) +} + +func (e setupEnvironment) consentState(read func(setupConsentState) bool) string { + if e.consent == nil { + return consentUnknown + } + if read(*e.consent) { + return consentActive + } + return consentMissing +} + +func defaultFileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + +// setupVoyageManifestPath is the path setup recommends for the Voyage +// capability manifest, so a re-run can enable the visual lane once the probe +// has written it. +func setupVoyageManifestPath(cfg *config.Config) string { + return filepath.Join(cfg.HomeDir, setupVoyageManifestName) +} + +// setupMistralManifestPath is the recommended Mistral capability manifest path. +func setupMistralManifestPath(cfg *config.Config) string { + return filepath.Join(cfg.HomeDir, setupMistralManifestName) +} + +// readSetupConsentState opens the archive read-only and reads every consent +// the report shows. Failures return nil so a report never blocks on the +// archive; the caller renders those consents as unknown. +func readSetupConsentState(ctx context.Context, cfg *config.Config) *setupConsentState { + if cfg == nil { + return nil + } + st, err := store.OpenReadOnly(cfg.DatabaseDSN()) + if err != nil { + return nil + } + defer func() { _ = st.Close() }() + return setupConsentFromStore(ctx, cfg, st) +} + +// setupConsentFromStore reads the consent records behind the report from an +// already-open store. Each lookup is independent so one failing table cannot +// hide the others. +func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.Store) *setupConsentState { + state := &setupConsentState{} + if consented, err := st.HasActiveDocumentProviderConsent(ctx); err == nil { + state.Documents = consented + } + if generation, err := st.ActiveVisualGeneration(ctx); err == nil && generation.Consented { + state.Visual = true + } else if generation, err := st.BuildingVisualGeneration(ctx); err == nil && generation.Consented { + state.Visual = true + } + if cfg.People.Sweep.Enabled { + if profile, err := cfg.People.Sweep.Profile(); err == nil { + if active, err := st.HasActivePersonInferenceConsent(ctx, profile.Fingerprint); err == nil { + state.PersonInference = active + } + } + } + if cfg.Vector.Enabled && cfg.Vector.People.Enabled { + if profile, err := cfg.Vector.SemanticPersonEmbeddingProfile(); err == nil { + if active, err := st.HasActivePersonSemanticEmbeddingConsent(ctx, profile.Fingerprint); err == nil { + state.PersonSemantic = active + } + } + } + return state +} + +// embeddingProviderName names the embedding destination for the report. +func embeddingProviderName(endpoint string) string { + lower := strings.ToLower(endpoint) + switch { + case strings.Contains(lower, "voyageai.com"): + return "voyage" + case strings.Contains(lower, "openai.com"): + return "openai" + case strings.Contains(lower, "localhost") || strings.Contains(lower, "127.0.0.1") || strings.Contains(lower, "::1"): + return "local" + case endpoint == "": + return "" + default: + return "custom" + } +} + +func embedScheduleSummary(schedule vector.EmbedScheduleConfig) string { + parts := []string{} + if schedule.Cron != "" { + parts = append(parts, "cron "+schedule.Cron) + } + if schedule.RunAfterSync { + parts = append(parts, "after each scheduled sync") + } + if len(parts) == 0 { + return "manual only" + } + return strings.Join(parts, ", ") +} + +// buildLaneReport derives the lane report from one loaded config plus the +// environment. It never contacts a provider. +func buildLaneReport(cfg *config.Config, env setupEnvironment) laneReport { + report := laneReport{ConfigPath: cfg.ConfigFilePath()} + report.Lanes = append(report.Lanes, + textSearchLane(cfg, env), + personSearchLane(cfg, env), + visualSearchLane(cfg, env), + documentsLane(cfg, env), + documentVectorsLane(cfg), + peopleInferenceLane(cfg, env), + activityLane(cfg), + mediaPolicyLane(cfg), + ) + report.MCPTools = liveMCPTools(cfg) + return report +} + +func textSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { + lane := laneStatus{Lane: laneTextSearch, Label: "Text search (messages, chats, meetings)"} + embeddings := cfg.Vector.Embeddings + if cfg.Vector.Enabled { + lane.State = laneStateOn + lane.Provider = embeddingProviderName(embeddings.Endpoint) + lane.Model = embeddings.Model + lane.Schedule = embedScheduleSummary(cfg.Vector.Embed.Schedule) + if embeddings.EffectiveAPIFormat() == vector.APIFormatVoyageContextual { + lane.Reason = "conversation windows and turn-aware meeting chunks share one contextual generation" + } else { + lane.Reason = "per-message vectors; no conversation-window context (Voyage contextual only)" + } + if embeddings.APIKeyEnv != "" && !env.hasEnv(embeddings.APIKeyEnv) { + lane.Reason += "; environment variable " + embeddings.APIKeyEnv + " is not set" + } + return lane + } + lane.State = laneStateOff + switch { + case embeddings.Endpoint != "" || embeddings.Model != "": + lane.Reason = "configured but disabled; set [vector] enabled = true" + case env.hasEnv(setupVoyageKeyEnv) || env.hasEnv(setupOpenAIKeyEnv): + lane.State = laneStatePending + lane.Reason = "an embedding key is present but the lane is not configured" + lane.Next = []string{"msgvault setup providers"} + default: + lane.Reason = "no embedding provider; set " + setupVoyageKeyEnv + " (recommended) or " + + setupOpenAIKeyEnv + ", or run Ollama with " + setupOllamaTextModel + ", then run `msgvault setup providers`" + } + return lane +} + +func personSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { + lane := laneStatus{Lane: lanePersonSearch, Label: "Semantic people search"} + switch { + case cfg.Vector.Enabled && cfg.Vector.People.Enabled: + lane.State = laneStateOn + lane.Provider = embeddingProviderName(cfg.Vector.Embeddings.Endpoint) + lane.Model = cfg.Vector.Embeddings.Model + lane.Consent = env.consentState(func(s setupConsentState) bool { return s.PersonSemantic }) + lane.Reason = "one curated document per person rides the text-search generation" + if lane.Consent != consentActive { + lane.Next = []string{"msgvault person provider consent --semantic-embeddings --yes"} + } + case cfg.Vector.Enabled: + lane.State = laneStateOff + lane.Reason = "set [vector.people] enabled = true with explicit retention and training postures" + default: + lane.State = laneStateOff + lane.Reason = "requires the text-search lane" + } + return lane +} + +func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { + lane := laneStatus{Lane: laneVisualSearch, Label: "Visual attachment search"} + multimodal := cfg.Vector.Multimodal + if multimodal.Enabled { + lane.State = laneStateOn + lane.Provider = multimodal.Provider + lane.Model = multimodal.Model + lane.Schedule = embedScheduleSummary(multimodal.Schedule) + lane.Consent = env.consentState(func(s setupConsentState) bool { return s.Visual }) + if !env.exists(multimodal.CapabilitiesFile) { + lane.State = laneStatePending + lane.Reason = "capabilities_file is missing; the daemon refuses every vector lane until it exists" + lane.Next = []string{visualProbeCommand(cfg)} + return lane + } + lane.Reason = "eligible images and short videos are embedded with bounded message context" + if lane.Consent != consentActive { + lane.Next = []string{"msgvault multimodal build --yes"} + } + return lane + } + lane.State = laneStateOff + if !env.hasEnv(multimodal.APIKeyEnv) { + lane.Reason = "needs " + multimodal.APIKeyEnv + " (Voyage is the only visual provider)" + return lane + } + lane.State = laneStatePending + if env.exists(setupVoyageManifestPath(cfg)) { + lane.Reason = "probe manifest found; setup can enable the lane" + lane.Next = []string{"msgvault setup providers"} + return lane + } + lane.Reason = "key present; the provider probe needs private synthetic WebP and MP4 seeds before uploads are authorized" + lane.Next = []string{visualProbeCommand(cfg), "msgvault setup providers"} + return lane +} + +func visualProbeCommand(cfg *config.Config) string { + return "msgvault multimodal probe --seeds --out " + setupVoyageManifestPath(cfg) + " --yes" +} + +func documentsLane(cfg *config.Config, env setupEnvironment) laneStatus { + lane := laneStatus{Lane: laneDocuments, Label: "Document attachments (extraction and lexical search)"} + documents := cfg.Attachments.Documents + manifest := setupMistralManifestPath(cfg) + if documents.Enabled { + lane.State = laneStateOn + lane.Provider = documents.Provider + lane.Model = documents.Model + lane.Consent = env.consentState(func(s setupConsentState) bool { return s.Documents }) + lane.Reason = fmt.Sprintf("region %s; retention=%s, training=%s; uploads are manual-only", + documents.Region, documents.RetentionPosture, documents.TrainingPosture) + if lane.Consent != consentActive { + if env.exists(manifest) { + lane.Next = []string{ + "msgvault documents consent-mistral --capabilities " + manifest + " --yes", + "msgvault documents build --capabilities " + manifest + " --yes", + } + } else { + lane.Next = []string{ + "msgvault documents probe-mistral --fixtures > " + manifest, + "msgvault documents consent-mistral --capabilities " + manifest + " --yes", + } + } + } + return lane + } + lane.State = laneStateOff + if env.hasEnv(documents.APIKeyEnv) { + lane.State = laneStatePending + lane.Reason = "key present but the lane is not configured" + lane.Next = []string{"msgvault setup providers"} + return lane + } + lane.Reason = "needs " + documents.APIKeyEnv + " (Mistral is the only document provider)" + return lane +} + +func documentVectorsLane(cfg *config.Config) laneStatus { + lane := laneStatus{Lane: laneDocumentVectors, Label: "Document semantic search"} + documents := cfg.Attachments.Documents + switch { + case documents.Enabled && documents.Index.Embeddings.Enabled: + lane.State = laneStateOn + lane.Provider = embeddingProviderName(cfg.Vector.Embeddings.Endpoint) + lane.Model = cfg.Vector.Embeddings.Model + lane.Schedule = embedScheduleSummary(cfg.Vector.Embed.Schedule) + lane.Reason = "document chunks are embedded with the text-search profile after a separate consent" + lane.Next = []string{"msgvault documents vectors consent --yes"} + case documents.Enabled && !cfg.Vector.Enabled: + lane.State = laneStateOff + lane.Reason = "requires the text-search lane" + case documents.Enabled: + lane.State = laneStateOff + lane.Reason = "set [attachments.documents.index.embeddings] enabled = true" + default: + lane.State = laneStateOff + lane.Reason = "requires the document lane" + } + return lane +} + +func peopleInferenceLane(cfg *config.Config, env setupEnvironment) laneStatus { + lane := laneStatus{Lane: lanePeopleInference, Label: "People sweep (attribute maintenance)"} + sweep := cfg.People.Sweep + if sweep.Enabled { + lane.State = laneStateOn + name, provider, err := sweep.ActiveProviderConfig() + if err == nil { + lane.Provider = name + " (" + string(provider.Protocol) + ")" + lane.Model = provider.Model + } + lane.Schedule = "cron " + sweep.Schedule + lane.Consent = env.consentState(func(s setupConsentState) bool { return s.PersonInference }) + lane.Reason = "runs for tracked people only; deterministic contact state refreshes for everyone through the activity job" + if lane.Consent != consentActive && name != "" { + lane.Next = []string{"msgvault person provider consent " + name + " --yes"} + } + lane.Next = append(lane.Next, "msgvault person track ") + return lane + } + lane.State = laneStateOff + if env.hasEnv(setupOpenAIKeyEnv) { + lane.State = laneStatePending + lane.Reason = setupOpenAIKeyEnv + " present; setup can onboard the " + setupInferenceModel + " profile" + lane.Next = []string{"msgvault setup providers"} + return lane + } + lane.Reason = "needs " + setupOpenAIKeyEnv + " or a reachable local Ollama server, then `msgvault setup providers`" + return lane +} + +func activityLane(cfg *config.Config) laneStatus { + lane := laneStatus{Lane: laneActivity, Label: "Contact activity (last contacted, cadence)"} + if cfg.Activity.Schedule == "" { + lane.State = laneStateOff + lane.Reason = "[activity] schedule is empty; run `msgvault activity build` by hand" + return lane + } + lane.State = laneStateOn + lane.Schedule = "cron " + cfg.Activity.Schedule + lane.Reason = "projects archived messages into dated per-person contact state (" + cfg.Activity.Timezone + ")" + return lane +} + +func mediaPolicyLane(cfg *config.Config) laneStatus { + lane := laneStatus{Lane: laneMediaPolicy, Label: "Chat media collection", State: laneStateOn} + summaries := []string{ + "beeper " + mediaPolicySummary(cfg.Beeper.MediaPolicy("")), + "slack " + mediaPolicySummary(cfg.Slack.MediaPolicy("")), + "discord " + mediaPolicySummary(cfg.Discord.MediaPolicy("")), + "teams " + mediaPolicySummary(cfg.Teams.MediaPolicy("")), + } + lane.Reason = strings.Join(summaries, "; ") + return lane +} + +func mediaPolicySummary(policy attachmentpolicy.Policy) string { + if policy.DisabledReason != "" { + return "off" + } + participants := "any size room" + if policy.MaxParticipants > 0 { + participants = fmt.Sprintf("rooms up to %d participants", policy.MaxParticipants) + } + size := "no size cap" + if policy.MaxBytes > 0 { + size = fmt.Sprintf("%d MiB cap", policy.MaxBytes>>20) + } + return fmt.Sprintf("scope %s, %s, %s", policy.Scope, participants, size) +} + +func liveMCPTools(cfg *config.Config) []string { + tools := []string{"search_people", "get_person_notes", "get_person_relationship", "search_person_files"} + if cfg.Vector.Enabled { + tools = append(tools, "semantic_search_messages", "find_similar_messages") + } + if cfg.Vector.Multimodal.Enabled { + tools = append(tools, "search_visual_attachments") + } + if cfg.Attachments.Documents.Enabled { + tools = append(tools, "search_document_attachments") + } + return tools +} + +func writeLaneReport(w io.Writer, report laneReport, jsonOutput bool) error { + if jsonOutput { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(report) + } + table := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(table, "LANE\tSTATE\tPROVIDER\tMODEL\tCONSENT\tSCHEDULE") + for _, lane := range report.Lanes { + _, _ = fmt.Fprintf(table, "%s\t%s\t%s\t%s\t%s\t%s\n", + lane.Label, lane.State, dash(lane.Provider), dash(lane.Model), dash(lane.Consent), dash(lane.Schedule)) + } + if err := table.Flush(); err != nil { + return fmt.Errorf("write lane report: %w", err) + } + _, _ = fmt.Fprintln(w) + for _, lane := range report.Lanes { + if lane.Reason == "" && len(lane.Next) == 0 { + continue + } + _, _ = fmt.Fprintf(w, "%s: %s\n", lane.Label, lane.Reason) + for _, next := range lane.Next { + _, _ = fmt.Fprintf(w, " next: %s\n", next) + } + } + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintf(w, "MCP tools live with this configuration: %s\n", strings.Join(report.MCPTools, ", ")) + _, _ = fmt.Fprintf(w, "Config: %s\n", report.ConfigPath) + return nil +} + +func dash(value string) string { + if value == "" { + return "-" + } + return value +} diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go new file mode 100644 index 000000000..5581443eb --- /dev/null +++ b/cmd/msgvault/cmd/setup_providers.go @@ -0,0 +1,1092 @@ +package cmd + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/charmbracelet/x/term" + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/documentindex" + "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/store" +) + +const ( + planActionEnable = "enable" + planActionKeep = "keep" + planActionPending = "pending" + planActionSkip = "skip" + planActionOnboard = "onboard" + + // Consent gates: one explicit answer per hosted provider. + gateVoyage = "voyage" + gateMistral = "mistral" + gateOpenAI = "openai" + + ollamaProbeTimeout = 2 * time.Second + ollamaProbeMaxBody = 1 << 20 + + tomlTableVector = "vector" +) + +// setupProvidersOptions are the command flags. +type setupProvidersOptions struct { + yes bool + dryRun bool + jsonOutput bool + documentRetention string + documentTraining string + retentionPosture string + trainingPosture string +} + +// ollamaProbeResult is what a local Ollama server reports about itself. +type ollamaProbeResult struct { + Reachable bool + Models []string +} + +// setupProvidersDeps isolates the pass from the process for tests: the +// environment, the config file, the archive, the daemon, and the people +// provider onboarding machinery are all injectable. +type setupProvidersDeps struct { + lookupEnv func(string) (string, bool) + fileExists func(string) bool + readConfigFile func() (config.ConfigFile, error) + editConfigTables func(string, []config.TableEdit) (config.ConfigFile, error) + loadConfig func(config.ConfigFile) (*config.Config, error) + remoteConfigured func() bool + isTerminal func(*cobra.Command) bool + probeOllama func(context.Context, string) ollamaProbeResult + consentState func(context.Context, *config.Config) *setupConsentState + daemonAlive func(context.Context, *config.Config) bool + personProvider func() personProviderCommandDeps + now func() time.Time +} + +func defaultSetupProvidersDeps() setupProvidersDeps { + return setupProvidersDeps{ + lookupEnv: os.LookupEnv, + fileExists: defaultFileExists, + readConfigFile: func() (config.ConfigFile, error) { + if cfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.ReadConfigFile(cfg.ConfigFilePath()) + }, + editConfigTables: func(ifMatch string, edits []config.TableEdit) (config.ConfigFile, error) { + if cfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.EditConfigTables(cfg.ConfigFilePath(), ifMatch, edits) + }, + loadConfig: func(snapshot config.ConfigFile) (*config.Config, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } + return loadSetupConfig(snapshot, cfg.HomeDir) + }, + remoteConfigured: IsRemoteMode, + isTerminal: commandStdinIsTerminal, + probeOllama: probeOllamaServer, + consentState: readSetupConsentState, + daemonAlive: func(ctx context.Context, loaded *config.Config) bool { + return findAnyDaemonRuntimeContext(ctx, loaded.Data.DataDir) != nil + }, + personProvider: defaultPersonProviderCommandDeps, + now: time.Now, + } +} + +// loadSetupConfig decodes a snapshot the way the daemon would, and keeps the +// operator's home directory when the file does not exist yet so recommended +// manifest paths land beside the config that setup is about to create. +func loadSetupConfig(snapshot config.ConfigFile, homeDir string) (*config.Config, error) { + loaded, err := config.LoadConfigFile(snapshot, homeDir) + if err != nil { + return nil, err + } + if !snapshot.Exists && homeDir != "" { + loaded.HomeDir = homeDir + loaded.Data.DataDir = homeDir + } + return loaded, nil +} + +func commandStdinIsTerminal(command *cobra.Command) bool { + file, ok := command.InOrStdin().(*os.File) + return ok && term.IsTerminal(file.Fd()) +} + +// probeOllamaServer lists the models a local Ollama server exposes. It is +// consulted only when no hosted embedding key is present; a failure simply +// reports the server as unreachable. +func probeOllamaServer(ctx context.Context, server string) ollamaProbeResult { + server = strings.TrimRight(strings.TrimSpace(server), "/") + if server == "" { + return ollamaProbeResult{} + } + ctx, cancel := context.WithTimeout(ctx, ollamaProbeTimeout) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, server+"/api/tags", nil) + if err != nil { + return ollamaProbeResult{} + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return ollamaProbeResult{} + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return ollamaProbeResult{} + } + var payload struct { + Models []struct { + Name string `json:"name"` + } `json:"models"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, ollamaProbeMaxBody)).Decode(&payload); err != nil { + return ollamaProbeResult{Reachable: true} + } + result := ollamaProbeResult{Reachable: true} + for _, model := range payload.Models { + if name := strings.TrimSpace(model.Name); name != "" { + result.Models = append(result.Models, name) + } + } + return result +} + +func (r ollamaProbeResult) hasModel(name string) bool { + for _, model := range r.Models { + if model == name || strings.HasPrefix(model, name+":") { + return true + } + } + return false +} + +// setupDetection is everything the plan is keyed off: which keys exist, what +// the local Ollama server offers, which probe manifests are already written, +// and which vector backend the archive selects. +type setupDetection struct { + voyageKey bool + mistralKey bool + mistralKeyEnv string + openAIKey bool + ollama ollamaProbeResult + ollamaEndpoint string + ollamaLoopback bool + voyageManifest string + mistralManifest string + backend string +} + +func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setupProvidersDeps) setupDetection { + env := setupEnvironment{lookupEnv: deps.lookupEnv, fileExists: deps.fileExists} + detection := setupDetection{ + voyageKey: env.hasEnv(setupVoyageKeyEnv), + mistralKeyEnv: loaded.Attachments.Documents.APIKeyEnv, + openAIKey: env.hasEnv(setupOpenAIKeyEnv), + backend: "sqlite-vec", + } + detection.mistralKey = env.hasEnv(detection.mistralKeyEnv) + if store.IsPostgresURL(loaded.DatabaseDSN()) { + detection.backend = "pgvector" + } + if path := loaded.Vector.Multimodal.CapabilitiesFile; path != "" && env.exists(path) { + detection.voyageManifest = path + } else if path := setupVoyageManifestPath(loaded); env.exists(path) { + detection.voyageManifest = path + } + if path := setupMistralManifestPath(loaded); env.exists(path) { + detection.mistralManifest = path + } + // The local server is consulted only when nothing hosted is available + // for the lane it would fill, so a key holder never waits on a probe. + textUnconfigured := !loaded.Vector.Enabled && loaded.Vector.Embeddings.Endpoint == "" + needsLocalText := textUnconfigured && !detection.voyageKey && !detection.openAIKey + needsLocalInference := !loaded.People.Sweep.Enabled && !detection.openAIKey + if (needsLocalText || needsLocalInference) && deps.probeOllama != nil { + server := strings.TrimRight(strings.TrimSpace(loaded.Chat.Server), "/") + detection.ollama = deps.probeOllama(ctx, server) + detection.ollamaEndpoint = server + "/v1" + if parsed, err := url.Parse(server); err == nil { + host := parsed.Hostname() + detection.ollamaLoopback = strings.EqualFold(host, "localhost") || host == "127.0.0.1" || host == "::1" + } + } + return detection +} + +// setupLanePlan is one lane's decision. The edits and next steps are +// attached so the plan can be printed before anything is written. +type setupLanePlan struct { + Lane string `json:"lane"` + Label string `json:"label"` + Action string `json:"action"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + Reason string `json:"reason"` + Gate string `json:"consent_gate,omitempty"` + edits []config.TableEdit + next []string +} + +// setupInferencePlan onboards one people-sweep provider profile through the +// same add, check, consent, and use path the CLI exposes. +type setupInferencePlan struct { + name string + options personProviderAddOptions + gate string +} + +type setupProvidersPlan struct { + Lanes []setupLanePlan + inference *setupInferencePlan +} + +func (p *setupProvidersPlan) laneOn(lane string) bool { + for _, item := range p.Lanes { + if item.Lane == lane { + return item.Action == planActionEnable || item.Action == planActionKeep || item.Action == planActionOnboard + } + } + return false +} + +func (p *setupProvidersPlan) gates() []string { + seen := map[string]bool{} + for _, lane := range p.Lanes { + if lane.Gate != "" && (lane.Action == planActionEnable || lane.Action == planActionOnboard) { + seen[lane.Gate] = true + } + } + ordered := []string{} + for _, gate := range []string{gateVoyage, gateMistral, gateOpenAI} { + if seen[gate] { + ordered = append(ordered, gate) + } + } + return ordered +} + +func (p *setupProvidersPlan) writes() bool { + for _, lane := range p.Lanes { + if len(lane.edits) > 0 || lane.Action == planActionOnboard { + return true + } + } + return false +} + +// declineGate turns every lane behind a declined gate into a skip. +func (p *setupProvidersPlan) declineGate(gate string) { + for i := range p.Lanes { + if p.Lanes[i].Gate == gate && (p.Lanes[i].Action == planActionEnable || p.Lanes[i].Action == planActionOnboard) { + p.Lanes[i].Action = planActionSkip + p.Lanes[i].Reason = "declined" + p.Lanes[i].edits = nil + p.Lanes[i].next = nil + } + } + if p.inference != nil && p.inference.gate == gate { + p.inference = nil + } +} + +// mergedEdits collapses the plan into one table edit per path so a single +// ETag-guarded write publishes every lane together. +func (p *setupProvidersPlan) mergedEdits() []config.TableEdit { + var merged []config.TableEdit + index := map[string]int{} + for _, lane := range p.Lanes { + if lane.Action != planActionEnable && lane.Action != planActionPending { + continue + } + for _, edit := range lane.edits { + key := strings.Join(edit.Path, "\x00") + if at, ok := index[key]; ok { + maps.Copy(merged[at].Values, edit.Values) + continue + } + values := make(map[string]any, len(edit.Values)) + maps.Copy(values, edit.Values) + index[key] = len(merged) + merged = append(merged, config.TableEdit{Path: edit.Path, Values: values}) + } + } + return merged +} + +func validateSetupProvidersOptions(options setupProvidersOptions) error { + if options.documentRetention != documentindex.RetentionStandard && options.documentRetention != documentindex.RetentionZDR { + return fmt.Errorf("--document-retention must be %q or %q", documentindex.RetentionStandard, documentindex.RetentionZDR) + } + if options.documentTraining != documentindex.TrainingDefaultOptOut && options.documentTraining != documentindex.TrainingOptedOut { + return fmt.Errorf("--document-training must be %q or %q", documentindex.TrainingDefaultOptOut, documentindex.TrainingOptedOut) + } + for name, value := range map[string]string{ + "--retention-posture": options.retentionPosture, "--training-posture": options.trainingPosture, + } { + if strings.TrimSpace(value) == "" || strings.EqualFold(strings.TrimSpace(value), "unknown") { + return fmt.Errorf("%s must be an explicit provider assertion", name) + } + } + return nil +} + +// planSetupProviders chooses values for every lane that is still unset. It +// never changes a lane that is already on: a configured text lane keeps its +// model even when a different key appears, because switching the embedding +// policy invalidates the index and is the operator's call. +func planSetupProviders(loaded *config.Config, detection setupDetection, options setupProvidersOptions, now time.Time) setupProvidersPlan { + // Later lanes read the plan so far: people search and document vectors + // depend on the text lane, and the sweep's evidence sources depend on the + // document lane, so the lanes are appended in dependency order. + plan := setupProvidersPlan{Lanes: []setupLanePlan{planTextSearch(loaded, detection)}} + person := planPersonSearch(loaded, &plan, options) + visual := planVisualSearch(loaded, detection) + documents := planDocuments(loaded, detection, options) + plan.Lanes = append(plan.Lanes, person, visual, documents) + vectors := planDocumentVectors(loaded, &plan) + inferenceLane, inference := planPeopleInference(loaded, detection, &plan, options, now) + plan.Lanes = append(plan.Lanes, vectors, inferenceLane) + plan.inference = inference + return plan +} + +func textLaneGate(plan *setupProvidersPlan) string { + for _, lane := range plan.Lanes { + if lane.Lane == laneTextSearch { + if lane.Action == planActionKeep { + return textLaneGateForProvider(lane.Provider) + } + return lane.Gate + } + } + return "" +} + +func textLaneGateForProvider(provider string) string { + switch provider { + case "voyage": + return gateVoyage + case "openai": + return gateOpenAI + default: + return "" + } +} + +func embedScheduleEdit(loaded *config.Config) []config.TableEdit { + schedule := loaded.Vector.Embed.Schedule + if schedule.Cron != "" || schedule.RunAfterSync { + return nil + } + return []config.TableEdit{{ + Path: []string{tomlTableVector, "embed", "schedule"}, + Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}, + }} +} + +func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePlan { + lane := setupLanePlan{Lane: laneTextSearch, Label: "Text search"} + embeddings := loaded.Vector.Embeddings + switch { + case loaded.Vector.Enabled: + lane.Action = planActionKeep + lane.Provider = embeddingProviderName(embeddings.Endpoint) + lane.Model = embeddings.Model + lane.Reason = "already configured; switching the embedding policy requires `msgvault embeddings build --full-rebuild`" + return lane + case embeddings.Endpoint != "" || embeddings.Model != "": + lane.Action = planActionSkip + lane.Reason = "configured but disabled; set [vector] enabled = true to turn it on" + return lane + } + vectorEdit := config.TableEdit{Path: []string{tomlTableVector}, Values: map[string]any{"enabled": true, "backend": detection.backend}} + switch { + case detection.voyageKey: + lane.Action, lane.Provider, lane.Model, lane.Gate = planActionEnable, "voyage", setupVoyageTextModel, gateVoyage + lane.Reason = "contextual embeddings: chats embed as conversation windows, meetings as turn-aware chunks, email on the same generation" + lane.edits = append([]config.TableEdit{vectorEdit, { + Path: []string{tomlTableVector, "embeddings"}, + Values: map[string]any{ + "api_format": "voyage-contextual", "endpoint": setupVoyageEndpoint, + "api_key_env": setupVoyageKeyEnv, "model": setupVoyageTextModel, "dimension": setupVoyageTextDim, + }, + }}, embedScheduleEdit(loaded)...) + case detection.openAIKey: + lane.Action, lane.Provider, lane.Model, lane.Gate = planActionEnable, "openai", setupOpenAITextModel, gateOpenAI + lane.Reason = "per-message vectors; no conversation-window context and no visual lane, both are Voyage-only" + lane.edits = append([]config.TableEdit{vectorEdit, { + Path: []string{tomlTableVector, "embeddings"}, + Values: map[string]any{ + "api_format": "openai", "endpoint": setupOpenAIEndpoint, + "api_key_env": setupOpenAIKeyEnv, "model": setupOpenAITextModel, "dimension": setupOpenAITextDim, + }, + }}, embedScheduleEdit(loaded)...) + case detection.ollama.Reachable && !detection.ollamaLoopback: + // A reachable server that is not on this machine would receive + // message text without a credential or a disclosure; only the operator + // may configure that, explicitly, in [vector.embeddings]. + lane.Action = planActionSkip + lane.Reason = "Ollama at " + detection.ollamaEndpoint + " is not loopback; setup only selects a local server, configure [vector.embeddings] explicitly to send text off this machine" + return lane + case detection.ollama.Reachable && detection.ollama.hasModel(setupOllamaTextModel): + lane.Action, lane.Provider, lane.Model = planActionEnable, "local", setupOllamaTextModel + lane.Reason = "local Ollama embeddings; message text stays on this machine" + lane.edits = append([]config.TableEdit{vectorEdit, { + Path: []string{tomlTableVector, "embeddings"}, + Values: map[string]any{ + "api_format": "openai", "endpoint": detection.ollamaEndpoint, + "model": setupOllamaTextModel, "dimension": setupOllamaTextDim, + "document_prefix": setupOllamaDocPrefix, "query_prefix": setupOllamaQueryPrefix, + "max_input_chars": setupOllamaMaxInput, + }, + }}, embedScheduleEdit(loaded)...) + case detection.ollama.Reachable: + lane.Action = planActionSkip + lane.Reason = "Ollama is reachable but has no " + setupOllamaTextModel + "; run `ollama pull " + setupOllamaTextModel + "` or set " + setupVoyageKeyEnv + return lane + default: + lane.Action = planActionSkip + lane.Reason = "no embedding provider: set " + setupVoyageKeyEnv + " (recommended) or " + setupOpenAIKeyEnv + ", or run Ollama with " + setupOllamaTextModel + return lane + } + lane.next = []string{"msgvault embeddings build --yes"} + return lane +} + +func planPersonSearch(loaded *config.Config, plan *setupProvidersPlan, options setupProvidersOptions) setupLanePlan { + lane := setupLanePlan{Lane: lanePersonSearch, Label: "Semantic people search"} + switch { + case loaded.Vector.Enabled && loaded.Vector.People.Enabled: + lane.Action = planActionKeep + lane.Reason = "already enabled" + case !plan.laneOn(laneTextSearch): + lane.Action = planActionSkip + lane.Reason = "requires the text-search lane" + default: + lane.Action = planActionEnable + lane.Gate = textLaneGate(plan) + lane.Reason = "one curated, non-sensitive attribute document per person rides the text-search generation" + lane.edits = []config.TableEdit{{ + Path: []string{tomlTableVector, "people"}, + Values: map[string]any{ + "enabled": true, "retention_posture": options.retentionPosture, "training_posture": options.trainingPosture, + }, + }} + lane.next = []string{"msgvault person provider consent --semantic-embeddings --yes"} + } + return lane +} + +func planVisualSearch(loaded *config.Config, detection setupDetection) setupLanePlan { + lane := setupLanePlan{Lane: laneVisualSearch, Label: "Visual attachment search", Provider: "voyage", Model: loaded.Vector.Multimodal.Model} + switch { + case loaded.Vector.Multimodal.Enabled: + lane.Action = planActionKeep + lane.Reason = "already enabled" + case !detection.voyageKey: + lane.Action = planActionSkip + lane.Provider, lane.Model = "", "" + lane.Reason = "needs " + setupVoyageKeyEnv + case detection.voyageManifest != "": + lane.Action, lane.Gate = planActionEnable, gateVoyage + lane.Reason = "probe manifest found at " + detection.voyageManifest + lane.edits = []config.TableEdit{ + {Path: []string{tomlTableVector, "multimodal"}, Values: map[string]any{"enabled": true, "capabilities_file": detection.voyageManifest}}, + {Path: []string{tomlTableVector, "multimodal", "schedule"}, Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}}, + } + lane.next = []string{"msgvault multimodal build --yes"} + default: + lane.Action = planActionPending + lane.Reason = "the provider probe needs private synthetic WebP and MP4 seeds; the lane stays off until the manifest exists" + lane.edits = []config.TableEdit{ + {Path: []string{tomlTableVector, "multimodal", "schedule"}, Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}}, + } + lane.next = []string{visualProbeCommand(loaded), "msgvault setup providers"} + } + return lane +} + +func planDocuments(loaded *config.Config, detection setupDetection, options setupProvidersOptions) setupLanePlan { + documents := loaded.Attachments.Documents + lane := setupLanePlan{Lane: laneDocuments, Label: "Document attachments", Provider: documents.Provider, Model: documents.Model} + manifest := setupMistralManifestPath(loaded) + switch { + case documents.Enabled: + lane.Action = planActionKeep + lane.Reason = "already enabled" + case !detection.mistralKey: + lane.Action = planActionSkip + lane.Provider, lane.Model = "", "" + lane.Reason = "needs " + detection.mistralKeyEnv + default: + lane.Action, lane.Gate = planActionEnable, gateMistral + lane.Reason = fmt.Sprintf("EU endpoint, %s; recorded postures retention=%s, training=%s (override with --document-retention/--document-training)", + documents.Model, options.documentRetention, options.documentTraining) + lane.edits = []config.TableEdit{{ + Path: []string{"attachments", "documents"}, + Values: map[string]any{ + "enabled": true, "retention_posture": options.documentRetention, "training_posture": options.documentTraining, + }, + }} + if detection.mistralManifest != "" { + lane.next = []string{ + "msgvault documents consent-mistral --capabilities " + manifest + " --yes", + "msgvault documents build --capabilities " + manifest + " --yes", + } + } else { + lane.next = []string{ + "msgvault documents probe-mistral --fixtures > " + manifest, + "msgvault documents consent-mistral --capabilities " + manifest + " --yes", + "msgvault documents build --capabilities " + manifest + " --yes", + } + } + } + return lane +} + +func planDocumentVectors(loaded *config.Config, plan *setupProvidersPlan) setupLanePlan { + lane := setupLanePlan{Lane: laneDocumentVectors, Label: "Document semantic search"} + documents := loaded.Attachments.Documents + switch { + case documents.Enabled && documents.Index.Embeddings.Enabled: + lane.Action = planActionKeep + lane.Reason = "already enabled" + case !plan.laneOn(laneDocuments): + lane.Action = planActionSkip + lane.Reason = "requires the document lane" + case !plan.laneOn(laneTextSearch): + lane.Action = planActionSkip + lane.Reason = "requires the text-search lane" + default: + lane.Action = planActionEnable + lane.Gate = textLaneGate(plan) + lane.Reason = "extracted document chunks are embedded with the text-search profile after `documents vectors consent`" + lane.edits = []config.TableEdit{{ + Path: []string{"attachments", "documents", "index", "embeddings"}, Values: map[string]any{"enabled": true}, + }} + lane.next = []string{"msgvault documents vectors consent --yes"} + } + return lane +} + +func planPeopleInference( + loaded *config.Config, + detection setupDetection, + plan *setupProvidersPlan, + options setupProvidersOptions, + now time.Time, +) (setupLanePlan, *setupInferencePlan) { + lane := setupLanePlan{Lane: lanePeopleInference, Label: "People sweep"} + if loaded.People.Sweep.Enabled { + lane.Action = planActionKeep + if name, provider, err := loaded.People.Sweep.ActiveProviderConfig(); err == nil { + lane.Provider, lane.Model = name, provider.Model + } + lane.Reason = "already enabled" + return lane, nil + } + sources := []string{string(peoplesweep.SourceConversationText), string(peoplesweep.SourceMeetingText)} + if plan.laneOn(laneDocuments) { + sources = append(sources, string(peoplesweep.SourceDocumentText)) + } + since := time.Date(now.Year()-1, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.DateOnly) + base := personProviderAddOptions{ + custom: true, protocol: string(peoplesweep.ProtocolOpenAIChat), + retentionPosture: options.retentionPosture, trainingPosture: options.trainingPosture, + allowedSources: sources, sourceSince: since, allowSensitive: true, + requestTimeout: time.Minute, confirmed: true, + } + switch { + case detection.openAIKey: + if _, exists := loaded.People.Sweep.Providers[setupInferenceProfile]; exists { + lane.Action = planActionSkip + lane.Provider = setupInferenceProfile + lane.Reason = "profile exists but the sweep is off; run `msgvault person provider consent " + + setupInferenceProfile + " --yes` and `msgvault person provider use " + setupInferenceProfile + "`" + return lane, nil + } + base.endpoint, base.model, base.auth = setupOpenAIEndpoint, setupInferenceModel, string(peoplesweep.AuthBearer) + base.credentialEnv, base.reasoningEffort = setupOpenAIKeyEnv, setupInferenceReasoning + lane.Action, lane.Provider, lane.Model, lane.Gate = planActionOnboard, setupInferenceProfile, setupInferenceModel, gateOpenAI + lane.Reason = fmt.Sprintf("openai_chat profile %q at %s reasoning; evidence from %s since %s; extraction runs for tracked people only", + setupInferenceProfile, setupInferenceReasoning, strings.Join(sources, ", "), since) + lane.next = []string{"msgvault person track "} + return lane, &setupInferencePlan{name: setupInferenceProfile, options: base, gate: gateOpenAI} + case detection.ollama.Reachable && detection.ollamaLoopback && detection.ollama.hasModel(loaded.Chat.Model): + if _, exists := loaded.People.Sweep.Providers[setupOllamaProfile]; exists { + lane.Action = planActionSkip + lane.Provider = setupOllamaProfile + lane.Reason = "profile exists but the sweep is off; run `msgvault person provider consent " + + setupOllamaProfile + " --yes` and `msgvault person provider use " + setupOllamaProfile + "`" + return lane, nil + } + // Only the configured chat model is eligible: picking whatever the + // server lists could select an embedding-only model whose check fails + // after the profile is already published. + model := loaded.Chat.Model + base.endpoint, base.model, base.auth = detection.ollamaEndpoint, model, string(peoplesweep.AuthNone) + lane.Action, lane.Provider, lane.Model = planActionOnboard, setupOllamaProfile, model + lane.Reason = "local Ollama server at " + detection.ollamaEndpoint + "; evidence stays on this machine" + lane.next = []string{"msgvault person track "} + return lane, &setupInferencePlan{name: setupOllamaProfile, options: base} + case detection.ollama.Reachable && !detection.ollamaLoopback: + lane.Action = planActionSkip + lane.Reason = "Ollama at " + detection.ollamaEndpoint + " is not loopback; add an authenticated profile with `msgvault person provider add`" + case detection.ollama.Reachable: + lane.Action = planActionSkip + lane.Reason = "Ollama has no " + loaded.Chat.Model + "; run `ollama pull " + loaded.Chat.Model + + "` or set [chat].model to an available chat model, then re-run setup" + default: + lane.Action = planActionSkip + lane.Reason = "needs " + setupOpenAIKeyEnv + " or a local Ollama server" + } + return lane, nil +} + +// gateDisclosure states plainly what each hosted provider receives once the +// operator answers yes. +func gateDisclosure(gate string, plan *setupProvidersPlan) string { + var lines []string + switch gate { + case gateVoyage: + lines = append(lines, "Voyage AI ("+setupVoyageEndpoint+") receives:") + if plan.laneOn(laneTextSearch) && textLaneProvider(plan) == "voyage" { + lines = append(lines, " - message, chat, and meeting text for embeddings ("+setupVoyageTextModel+")") + } + if plan.laneOn(lanePersonSearch) && textLaneProvider(plan) == "voyage" { + lines = append(lines, " - one curated, non-sensitive attribute document per person") + } + if plan.laneOn(laneVisualSearch) { + lines = append(lines, " - eligible image and video attachment bytes with bounded message context, after `msgvault multimodal build --yes`") + } + if plan.laneOn(laneDocumentVectors) && textLaneProvider(plan) == "voyage" { + lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`") + } + case gateMistral: + lines = append(lines, + "Mistral (EU region, "+documentindex.ModelMistralOCR+") receives:", + " - complete original bytes of standalone document attachments, only after the probe manifest and `msgvault documents consent-mistral --yes`", + " - postures recorded now: retention="+planValue(plan, laneDocuments, "retention_posture")+", training="+planValue(plan, laneDocuments, "training_posture")) + case gateOpenAI: + lines = append(lines, "OpenAI ("+setupOpenAIEndpoint+") receives:") + if plan.laneOn(laneTextSearch) && textLaneProvider(plan) == "openai" { + lines = append(lines, " - message, chat, and meeting text for embeddings ("+setupOpenAITextModel+")") + } + if plan.laneOn(lanePersonSearch) && textLaneProvider(plan) == "openai" { + lines = append(lines, " - one curated, non-sensitive attribute document per person") + } + if plan.laneOn(laneDocumentVectors) && textLaneProvider(plan) == "openai" { + lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`") + } + if plan.inference != nil && plan.inference.gate == gateOpenAI { + lines = append(lines, " - bounded evidence packets of "+ + strings.Join(plan.inference.options.allowedSources, ", ")+ + " for tracked people ("+setupInferenceModel+"); a synthetic check request is sent now") + } + } + return strings.Join(lines, "\n") +} + +func textLaneProvider(plan *setupProvidersPlan) string { + for _, item := range plan.Lanes { + if item.Lane == laneTextSearch { + return item.Provider + } + } + return "" +} + +func planValue(plan *setupProvidersPlan, lane, key string) string { + for _, item := range plan.Lanes { + if item.Lane != lane { + continue + } + for _, edit := range item.edits { + if value, ok := edit.Values[key]; ok { + return fmt.Sprint(value) + } + } + } + return "" +} + +func writeSetupPlan(w io.Writer, configPath string, plan *setupProvidersPlan) { + _, _ = fmt.Fprintf(w, "Provider setup plan for %s\n", configPath) + table := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + for _, lane := range plan.Lanes { + _, _ = fmt.Fprintf(table, " %s\t%s\t%s\t%s\t%s\n", lane.Label, lane.Action, dash(lane.Provider), dash(lane.Model), lane.Reason) + } + _ = table.Flush() +} + +type setupProvidersOutput struct { + Plan []setupLanePlan `json:"plan"` + Applied bool `json:"applied"` + DryRun bool `json:"dry_run"` + Declined []string `json:"declined,omitempty"` + FollowUps []string `json:"follow_ups,omitempty"` + Report laneReport `json:"report"` +} + +func newSetupProvidersCommand(deps setupProvidersDeps) *cobra.Command { + var options setupProvidersOptions + command := &cobra.Command{ + Use: "providers", + Short: "Turn on the retrieval and people lanes the available API keys support, with recommended defaults", + Long: `Read the environment and configure every lane that is still unset: + + ` + setupVoyageKeyEnv + ` text search with Voyage contextual embeddings (conversation + windows for chats, turn-aware chunks for meetings), semantic + people search, and the visual attachment lane once its probe + manifest exists + ` + setupOpenAIKeyEnv + ` text search on the OpenAI-compatible path when no Voyage key + is present, and the people sweep on ` + setupInferenceModel + ` + MISTRAL_API_KEY document attachment extraction, plus document vectors when a + text lane is on + (no keys) a local Ollama server at [chat].server when it is reachable + +Hosted lanes never turn on from a key alone: setup asks once per provider, +writes the recommended values to config.toml, runs the people-provider +check and consent, and prints what is on, what is off, and why. Lanes that +are already configured are left alone, so re-running after adding a key +upgrades only that lane. Probe manifests are expected at +/` + setupVoyageManifestName + ` and /` + setupMistralManifestName + `.`, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + return runSetupProviders(command, deps, options) + }, + } + flags := command.Flags() + flags.BoolVar(&options.yes, "yes", false, "Accept every provider disclosure without prompting") + flags.BoolVar(&options.dryRun, "dry-run", false, "Print the plan and the current lane report without writing anything") + flags.BoolVar(&options.jsonOutput, flagJSON, false, "Output structured JSON") + flags.StringVar(&options.documentRetention, "document-retention", documentindex.RetentionStandard, + "Mistral retention posture to record: standard or zdr") + flags.StringVar(&options.documentTraining, "document-training", documentindex.TrainingDefaultOptOut, + "Mistral training posture to record: default-opt-out or opted-out") + flags.StringVar(&options.retentionPosture, "retention-posture", setupPostureDeclared, + "Retention assertion recorded for embedding and inference providers") + flags.StringVar(&options.trainingPosture, "training-posture", setupPostureDeclared, + "Training assertion recorded for embedding and inference providers") + return command +} + +func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options setupProvidersOptions) error { + if err := validateSetupProvidersOptions(options); err != nil { + return err + } + if deps.remoteConfigured != nil && deps.remoteConfigured() { + return errors.New("setup providers cannot run against a configured remote daemon: it edits this machine's config.toml, which the remote daemon never reads; run it on the daemon host, or pass --local to configure a daemon on this machine") + } + if deps.readConfigFile == nil || deps.editConfigTables == nil || deps.loadConfig == nil { + return errors.New("setup providers config editing is unavailable") + } + ctx := command.Context() + before, err := deps.readConfigFile() + if err != nil { + return err + } + loaded, err := deps.loadConfig(before) + if err != nil { + return err + } + now := time.Now() + if deps.now != nil { + now = deps.now() + } + detection := detectSetupProviders(ctx, loaded, deps) + plan := planSetupProviders(loaded, detection, options, now) + out := command.OutOrStdout() + if !options.jsonOutput { + writeSetupPlan(out, loaded.ConfigFilePath(), &plan) + _, _ = fmt.Fprintln(out) + } + + var declined []string + gates := plan.gates() + if options.dryRun { + if !options.jsonOutput { + for _, gate := range gates { + _, _ = fmt.Fprintln(out, gateDisclosure(gate, &plan)) + } + _, _ = fmt.Fprintln(out, "Dry run: nothing written.") + _, _ = fmt.Fprintln(out) + } + return writeSetupProvidersResult(command, deps, loaded, &plan, options, false, declined) + } + if len(gates) > 0 && !options.yes { + if deps.isTerminal == nil || !deps.isTerminal(command) || options.jsonOutput { + return fmt.Errorf("setup providers needs one consent per hosted provider (%s); review the plan with --dry-run, then re-run with --yes", + strings.Join(gates, ", ")) + } + reader := bufio.NewReader(command.InOrStdin()) + for _, gate := range gates { + _, _ = fmt.Fprintln(out, gateDisclosure(gate, &plan)) + _, _ = fmt.Fprintf(out, "Enable the %s lanes? [y/N]: ", gate) + answer, readErr := reader.ReadString('\n') + if readErr != nil && !errors.Is(readErr, io.EOF) { + return fmt.Errorf("read consent answer: %w", readErr) + } + if !isYesAnswer(strings.ToLower(strings.TrimSpace(answer))) { + declined = append(declined, gate) + plan.declineGate(gate) + } + _, _ = fmt.Fprintln(out) + } + } else if !options.jsonOutput { + for _, gate := range gates { + _, _ = fmt.Fprintln(out, gateDisclosure(gate, &plan)) + _, _ = fmt.Fprintf(out, "Accepted with --yes.\n\n") + } + } + + edits := plan.mergedEdits() + if len(edits) > 0 { + if err := config.ValidateConfigTableEdits(before, edits); err != nil { + return fmt.Errorf("planned config changes are invalid: %w", err) + } + if _, err := deps.editConfigTables(before.ETag, edits); err != nil { + return fmt.Errorf("write config: %w", err) + } + } + if plan.inference != nil && deps.personProvider != nil { + if err := onboardSetupInferenceProfile(command, deps.personProvider(), *plan.inference, options.jsonOutput); err != nil { + return err + } + } + return writeSetupProvidersResult(command, deps, nil, &plan, options, plan.writes(), declined) +} + +// writeSetupProvidersResult reloads the config (unless the caller passes the +// unchanged one) and prints the lane report plus follow-up commands. +func writeSetupProvidersResult( + command *cobra.Command, + deps setupProvidersDeps, + loaded *config.Config, + plan *setupProvidersPlan, + options setupProvidersOptions, + applied bool, + declined []string, +) error { + ctx := command.Context() + if loaded == nil { + snapshot, err := deps.readConfigFile() + if err != nil { + return err + } + loaded, err = deps.loadConfig(snapshot) + if err != nil { + return err + } + } + env := setupEnvironment{lookupEnv: deps.lookupEnv, fileExists: deps.fileExists} + if deps.consentState != nil { + env.consent = deps.consentState(ctx, loaded) + } + report := buildLaneReport(loaded, env) + followUps := setupFollowUps(ctx, deps, loaded, plan, applied) + if options.jsonOutput { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(setupProvidersOutput{ + Plan: plan.Lanes, Applied: applied, DryRun: options.dryRun, + Declined: declined, FollowUps: followUps, Report: report, + }) + } + out := command.OutOrStdout() + if applied { + _, _ = fmt.Fprintf(out, "Configuration written to %s\n\n", loaded.ConfigFilePath()) + } + if err := writeLaneReport(out, report, false); err != nil { + return err + } + if len(followUps) > 0 { + _, _ = fmt.Fprintln(out) + if options.dryRun { + _, _ = fmt.Fprintln(out, "Next steps after applying:") + } else { + _, _ = fmt.Fprintln(out, "Next steps:") + } + for i, step := range followUps { + _, _ = fmt.Fprintf(out, " %d. %s\n", i+1, step) + } + } + return nil +} + +// setupFollowUps orders the commands that finish what setup started: restart +// a running daemon first so it loads the new lanes, then consents and builds. +func setupFollowUps(ctx context.Context, deps setupProvidersDeps, loaded *config.Config, plan *setupProvidersPlan, applied bool) []string { + var steps []string + seen := map[string]bool{} + add := func(step string) { + if step != "" && !seen[step] { + seen[step] = true + steps = append(steps, step) + } + } + if applied && deps.daemonAlive != nil && deps.daemonAlive(ctx, loaded) { + add("msgvault daemon restart") + } + for _, lane := range plan.Lanes { + if lane.Action == planActionEnable || lane.Action == planActionPending || lane.Action == planActionOnboard { + for _, step := range lane.next { + add(step) + } + } + } + sort.SliceStable(steps, func(i, j int) bool { + // Probes and consents before builds, so the printed order is runnable. + return followUpRank(steps[i]) < followUpRank(steps[j]) + }) + return steps +} + +func followUpRank(step string) int { + switch { + case step == "msgvault daemon restart": + return 0 + case strings.Contains(step, " probe"): + return 1 + case strings.Contains(step, " consent"): + return 2 + case strings.Contains(step, "setup providers"): + return 3 + case strings.Contains(step, " build"): + return 4 + default: + return 5 + } +} + +// onboardSetupInferenceProfile publishes, checks, consents to, and selects +// one people-sweep profile through the same path `person provider add`, +// `consent`, and `use` take, so setup never bypasses a gate those commands +// enforce. Existing profiles are not re-added; a missing consent or +// selection is completed. +func onboardSetupInferenceProfile( + command *cobra.Command, + deps personProviderCommandDeps, + plan setupInferencePlan, + quiet bool, +) error { + if deps.readConfigFile == nil { + return errors.New("people provider config editing is unavailable") + } + // The add step's own summary tells the operator to run consent and use + // next, which setup does itself, so that step always runs silently; the + // consent disclosure and the selection notice are kept unless --json. + silent := silentSetupCommand(command) + target := command + if quiet { + target = silent + } + before, err := deps.readConfigFile() + if err != nil { + return err + } + sweep, err := personProviderConfigFromSnapshot(deps, before) + if err != nil { + return err + } + if _, exists := sweep.Providers[plan.name]; !exists { + if err := runPersonProviderAdd(silent, deps, plan.name, plan.options); err != nil { + return fmt.Errorf("onboard people provider %q: %w", plan.name, err) + } + if !quiet { + _, _ = fmt.Fprintf(command.OutOrStdout(), "Added and checked people provider profile %q.\n", plan.name) + } + } + after, err := deps.readConfigFile() + if err != nil { + return err + } + sweep, err = personProviderConfigFromSnapshot(deps, after) + if err != nil { + return err + } + selected, err := selectPersonProviderConfig(sweep, plan.name) + if err != nil { + return err + } + selected.Enabled = true + if err := consentSetupInferenceProfile(target, deps, plan.name, selected); err != nil { + return fmt.Errorf("consent to people provider %q: %w", plan.name, err) + } + if sweep.Enabled && sweep.Provider.Name == plan.name { + return nil + } + useDeps := deps + useDeps.config = func() peoplesweep.Config { return sweep } + if err := runPersonProviderUse(target, useDeps, plan.name, false); err != nil { + return fmt.Errorf("select people provider %q: %w", plan.name, err) + } + return nil +} + +// silentSetupCommand mirrors command's context and stdin with discarded +// standard output, for steps whose own summary would contradict the pass. +func silentSetupCommand(command *cobra.Command) *cobra.Command { + silent := &cobra.Command{Use: command.Use} + silent.SetContext(command.Context()) + silent.SetOut(io.Discard) + silent.SetErr(command.ErrOrStderr()) + silent.SetIn(command.InOrStdin()) + return silent +} + +// consentSetupInferenceProfile records consent directly when this process +// may write the archive, and otherwise proxies `person provider consent +// --yes` to the daemon that owns it. +func consentSetupInferenceProfile( + command *cobra.Command, + deps personProviderCommandDeps, + name string, + selected peoplesweep.Config, +) error { + directStore, _, err := personProviderMutationScope(command.Context(), deps) + if err != nil { + return err + } + if directStore { + consentDeps := deps + consentDeps.config = func() peoplesweep.Config { return selected } + return runPersonProviderConsent(command, consentDeps, true, false, false) + } + if deps.proxy == nil { + return errors.New("people provider daemon proxy is unavailable") + } + root := &cobra.Command{Use: "msgvault"} + person := &cobra.Command{Use: "person"} + provider := &cobra.Command{Use: personProviderCommandName} + leaf := &cobra.Command{Use: cmdUseConsent} + leaf.Flags().Bool("yes", false, "") + if err := leaf.Flags().Set("yes", "true"); err != nil { + return fmt.Errorf("set people provider consent flag: %w", err) + } + provider.AddCommand(leaf) + person.AddCommand(provider) + root.AddCommand(person) + leaf.SetOut(command.OutOrStdout()) + leaf.SetErr(command.ErrOrStderr()) + return deps.proxy(leaf, []string{name}, nil) +} diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go new file mode 100644 index 000000000..8e3ff1133 --- /dev/null +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -0,0 +1,584 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/documentindex" + "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" + "go.kenn.io/msgvault/internal/vector" +) + +const setupProvidersTestKey = "setup-providers-test-key" + +// setupProvidersFixture is one operator machine: a real config file, a real +// archive store, a fixed environment, and a fixed filesystem view for the +// probe manifests. +type setupProvidersFixture struct { + dir string + path string + store *store.Store + env map[string]string + files map[string]bool + ollama ollamaProbeResult + checker *fixedPersonProviderChecker + input io.Reader + tty bool +} + +func newSetupProvidersFixture(t *testing.T, content string) *setupProvidersFixture { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + if content != "" { + content = strings.ReplaceAll(content, "{{DIR}}", filepath.ToSlash(dir)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + return &setupProvidersFixture{ + dir: dir, + path: path, + store: testutil.NewSQLiteTestStore(t), + env: map[string]string{}, + files: map[string]bool{}, + checker: &fixedPersonProviderChecker{response: peoplesweep.StructuredResponse{ + Output: []byte(`{"ok":true}`), ProviderVersion: peoplesweep.OpenAIChatProviderVersion, + ModelVersion: "setup-model-v1", + }}, + } +} + +func (f *setupProvidersFixture) load(t *testing.T) *config.Config { + t.Helper() + snapshot, err := config.ReadConfigFile(f.path) + require.NoError(t, err) + loaded, err := loadSetupConfig(snapshot, f.dir) + require.NoError(t, err) + return loaded +} + +func (f *setupProvidersFixture) lookupEnv(name string) (string, bool) { + value, ok := f.env[name] + return value, ok +} + +func (f *setupProvidersFixture) personProviderDeps(t *testing.T) personProviderCommandDeps { + t.Helper() + loaded := f.load(t) + deps := localPersonProviderDeps(loaded.People.Sweep, f.store, f.checker) + deps.readConfigFile = func() (config.ConfigFile, error) { return config.ReadConfigFile(f.path) } + deps.configHomeDir = func() string { return f.dir } + deps.editConfigTables = func(etag string, edits []config.TableEdit) (config.ConfigFile, error) { + return config.EditConfigTables(f.path, etag, edits) + } + deps.restoreConfigFile = func(published, before config.ConfigFile) (config.ConfigFile, error) { + return config.RestoreConfigFile(f.path, published, before) + } + deps.setup = personProviderSetupDeps{ + lookupEnv: f.lookupEnv, + negotiate: func(_ context.Context, candidate peoplesweep.ProviderConfig, credential peoplesweep.Credential) (peoplesweep.NegotiatedCapabilities, error) { + if credential.Scheme != peoplesweep.AuthNone { + assert.Equal(t, setupProvidersTestKey, credential.Value()) + } + return peoplesweep.NegotiatedCapabilities{ + OutputMode: peoplesweep.OutputModeNativeJSONSchema, TokenLimitParameter: "max_completion_tokens", + ReasoningEffort: candidate.ReasoningEffort, DriverVersion: peoplesweep.OpenAIChatProviderVersion, + }, nil + }, + } + return deps +} + +func (f *setupProvidersFixture) deps(t *testing.T) setupProvidersDeps { + t.Helper() + return setupProvidersDeps{ + lookupEnv: f.lookupEnv, + fileExists: func(path string) bool { return f.files[path] }, + readConfigFile: func() (config.ConfigFile, error) { + return config.ReadConfigFile(f.path) + }, + editConfigTables: func(etag string, edits []config.TableEdit) (config.ConfigFile, error) { + return config.EditConfigTables(f.path, etag, edits) + }, + loadConfig: func(snapshot config.ConfigFile) (*config.Config, error) { + return loadSetupConfig(snapshot, f.dir) + }, + remoteConfigured: func() bool { return false }, + isTerminal: func(*cobra.Command) bool { return f.tty }, + probeOllama: func(context.Context, string) ollamaProbeResult { return f.ollama }, + consentState: func(ctx context.Context, loaded *config.Config) *setupConsentState { + return setupConsentFromStore(ctx, loaded, f.store) + }, + daemonAlive: func(context.Context, *config.Config) bool { return false }, + personProvider: func() personProviderCommandDeps { return f.personProviderDeps(t) }, + now: func() time.Time { return time.Date(2026, time.September, 3, 12, 0, 0, 0, time.UTC) }, + } +} + +func (f *setupProvidersFixture) run(t *testing.T, args ...string) (string, error) { + t.Helper() + root := &cobra.Command{Use: "msgvault"} + setup := &cobra.Command{Use: "setup"} + setup.AddCommand(newSetupProvidersCommand(f.deps(t))) + setup.AddCommand(newSetupStatusCommand(setupStatusDeps{ + config: func() *config.Config { return f.load(t) }, + environment: func(command *cobra.Command, loaded *config.Config) setupEnvironment { + return setupEnvironment{ + lookupEnv: f.lookupEnv, + fileExists: func(path string) bool { return f.files[path] }, + consent: setupConsentFromStore(command.Context(), loaded, f.store), + } + }, + })) + root.AddCommand(setup) + root.SetArgs(append([]string{"setup"}, args...)) + input := f.input + if input == nil { + input = strings.NewReader("") + } + root.SetIn(input) + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&output) + err := root.ExecuteContext(t.Context()) + return output.String(), err +} + +func (f *setupProvidersFixture) readConfig(t *testing.T) string { + t.Helper() + content, err := os.ReadFile(f.path) + if os.IsNotExist(err) { + return "" + } + require.NoError(t, err) + return string(content) +} + +const setupProvidersMinimalConfig = `# operator comment survives setup +[data] +data_dir = "{{DIR}}/data" +` + +func TestSetupProvidersVoyageWritesRecommendedDefaults(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.True(loaded.Vector.Enabled) + assert.Equal("sqlite-vec", loaded.Vector.Backend) + assert.Equal(vector.APIFormatVoyageContextual, loaded.Vector.Embeddings.EffectiveAPIFormat()) + assert.Equal(setupVoyageTextModel, loaded.Vector.Embeddings.Model) + assert.Equal(setupVoyageTextDim, loaded.Vector.Embeddings.Dimension) + assert.Equal(setupVoyageEndpoint, loaded.Vector.Embeddings.Endpoint) + assert.Equal(setupVoyageKeyEnv, loaded.Vector.Embeddings.APIKeyEnv) + assert.True(loaded.Vector.Embed.Schedule.RunAfterSync) + assert.Equal(setupEmbedCron, loaded.Vector.Embed.Schedule.Cron) + assert.True(loaded.Vector.People.Enabled) + assert.Equal(setupPostureDeclared, loaded.Vector.People.RetentionPosture) + assert.Equal(setupPostureDeclared, loaded.Vector.People.TrainingPosture) + // The visual lane stays off until the probe manifest exists: enabling it + // without one makes the daemon refuse every vector lane. + assert.False(loaded.Vector.Multimodal.Enabled) + assert.True(loaded.Vector.Multimodal.Schedule.RunAfterSync) + assert.False(loaded.Attachments.Documents.Enabled) + assert.False(loaded.People.Sweep.Enabled) + + assert.Contains(fixture.readConfig(t), "# operator comment survives setup") + assert.Contains(output, "Configuration written to") + assert.Contains(output, "multimodal probe --seeds --out "+setupVoyageManifestPath(loaded)) + assert.Contains(output, "msgvault embeddings build --yes") + assert.Contains(output, "person provider consent --semantic-embeddings --yes") + assert.NotContains(output, setupProvidersTestKey) + + // Re-running leaves a configured archive alone. + before := fixture.readConfig(t) + output, err = fixture.run(t, "providers", "--yes") + require.NoError(err, output) + assert.Equal(before, fixture.readConfig(t)) + assert.Contains(output, "already configured") +} + +func TestSetupProvidersCreatesMissingConfigFile(t *testing.T) { + require := require.New(t) + fixture := newSetupProvidersFixture(t, "") + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + loaded := fixture.load(t) + require.True(loaded.Vector.Enabled) + require.Equal(setupVoyageTextModel, loaded.Vector.Embeddings.Model) +} + +func TestSetupProvidersEnablesVisualLaneWhenManifestExists(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + manifest := filepath.Join(fixture.dir, setupVoyageManifestName) + fixture.files[manifest] = true + + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.True(loaded.Vector.Multimodal.Enabled) + assert.Equal(manifest, loaded.Vector.Multimodal.CapabilitiesFile) + assert.True(loaded.Vector.Multimodal.Schedule.RunAfterSync) + assert.Equal(setupEmbedCron, loaded.Vector.Multimodal.Schedule.Cron) + assert.Contains(output, "msgvault multimodal build --yes") +} + +func TestSetupProvidersMistralEnablesDocumentsAndVectors(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--yes", + "--document-retention", documentindex.RetentionZDR, "--document-training", documentindex.TrainingOptedOut) + require.NoError(err, output) + + loaded := fixture.load(t) + documents := loaded.Attachments.Documents + assert.True(documents.Enabled) + assert.Equal(documentindex.RetentionZDR, documents.RetentionPosture) + assert.Equal(documentindex.TrainingOptedOut, documents.TrainingPosture) + assert.Equal(documentindex.ModelMistralOCR, documents.Model) + assert.True(documents.Index.Embeddings.Enabled) + manifest := setupMistralManifestPath(loaded) + assert.Contains(output, "documents probe-mistral --fixtures > "+manifest) + assert.Contains(output, "documents consent-mistral --capabilities "+manifest+" --yes") + assert.Contains(output, "msgvault documents vectors consent --yes") + assert.Contains(output, "retention=zdr, training=opted-out") +} + +func TestSetupProvidersRejectsUnknownDocumentPostures(t *testing.T) { + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + + _, err := fixture.run(t, "providers", "--yes", "--document-retention", "unknown") + require.ErrorContains(t, err, "--document-retention") + assert.Empty(t, strings.TrimSpace(strings.TrimPrefix(fixture.readConfig(t), strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir))))) +} + +func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.True(loaded.Vector.Enabled) + assert.Equal(vector.APIFormatOpenAI, loaded.Vector.Embeddings.EffectiveAPIFormat()) + assert.Equal(setupOpenAITextModel, loaded.Vector.Embeddings.Model) + assert.Equal(setupOpenAITextDim, loaded.Vector.Embeddings.Dimension) + assert.Equal(setupOpenAIKeyEnv, loaded.Vector.Embeddings.APIKeyEnv) + assert.False(loaded.Vector.Multimodal.Enabled) + + sweep := loaded.People.Sweep + require.True(sweep.Enabled) + assert.Equal(setupInferenceProfile, sweep.Provider.Name) + profile := sweep.Providers[setupInferenceProfile] + assert.Equal(peoplesweep.ProtocolOpenAIChat, profile.Protocol) + assert.Equal(setupOpenAIEndpoint, profile.Endpoint) + assert.Equal(setupInferenceModel, profile.Model) + assert.Equal(setupInferenceReasoning, profile.ReasoningEffort) + assert.Equal(peoplesweep.CredentialEnv, profile.Credential) + assert.Equal(setupOpenAIKeyEnv, profile.CredentialEnv) + assert.Equal("2025-01-01", profile.SourceSince) + assert.True(profile.AllowSensitive) + assert.ElementsMatch([]peoplesweep.SourceClass{peoplesweep.SourceConversationText, peoplesweep.SourceMeetingText}, profile.AllowedSources) + + // The onboarding went through the real add, check, consent, and use gates. + fingerprint, err := sweep.Profile() + require.NoError(err) + checked, err := fixture.store.HasSuccessfulPersonInferenceCheck(t.Context(), fingerprint.Fingerprint) + require.NoError(err) + assert.True(checked) + consented, err := fixture.store.HasActivePersonInferenceConsent(t.Context(), fingerprint.Fingerprint) + require.NoError(err) + assert.True(consented) + assert.EqualValues(1, fixture.checker.calls.Load()) + + assert.Contains(output, "no conversation-window context") + assert.Contains(output, "msgvault person track ") + assert.NotContains(output, setupProvidersTestKey) + + // Status now reports the sweep on with an active consent. + status, err := fixture.run(t, "status", "--json") + require.NoError(err, status) + var report laneReport + require.NoError(json.Unmarshal([]byte(status), &report)) + inference := findLane(t, report, lanePeopleInference) + assert.Equal(laneStateOn, inference.State) + assert.Equal(consentActive, inference.Consent) + assert.Equal(setupInferenceModel, inference.Model) +} + +func TestSetupProvidersLocalOllamaFallback(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} + + output, err := fixture.run(t, "providers") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.True(loaded.Vector.Enabled) + assert.Equal("http://localhost:11434/v1", loaded.Vector.Embeddings.Endpoint) + assert.Equal(setupOllamaTextModel, loaded.Vector.Embeddings.Model) + assert.Equal(setupOllamaTextDim, loaded.Vector.Embeddings.Dimension) + assert.Equal(setupOllamaDocPrefix, loaded.Vector.Embeddings.DocumentPrefix) + assert.Equal(setupOllamaQueryPrefix, loaded.Vector.Embeddings.QueryPrefix) + assert.Equal(setupOllamaMaxInput, loaded.Vector.Embeddings.MaxInputChars) + assert.Empty(loaded.Vector.Embeddings.APIKeyEnv) + + sweep := loaded.People.Sweep + require.True(sweep.Enabled) + assert.Equal(setupOllamaProfile, sweep.Provider.Name) + profile := sweep.Providers[setupOllamaProfile] + assert.Equal("gpt-oss-128k", profile.Model) + assert.Equal(peoplesweep.AuthNone, profile.Auth) + assert.Equal(peoplesweep.CredentialNone, profile.Credential) + assert.Equal("http://localhost:11434/v1", profile.Endpoint) + assert.Contains(output, "stays on this machine") +} + +func TestSetupProvidersSkipsRemoteOllama(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` +[chat] +server = "http://ollama.internal.example:11434" +`) + fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} + before := fixture.readConfig(t) + + output, err := fixture.run(t, "providers", "--json") + require.NoError(err, output) + // A reachable server off this machine is never selected: it would receive + // message text without a credential or a disclosure. + assert.Equal(before, fixture.readConfig(t)) + var result setupProvidersOutput + require.NoError(json.Unmarshal([]byte(output), &result)) + for _, lane := range result.Plan { + assert.Equal(planActionSkip, lane.Action, lane.Lane) + if lane.Lane == laneTextSearch || lane.Lane == lanePeopleInference { + assert.Contains(lane.Reason, "not loopback") + } + } +} + +func TestSetupProvidersOllamaWithoutChatModelSkipsInference(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest"}} + + output, err := fixture.run(t, "providers") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.True(loaded.Vector.Enabled) + assert.Equal(setupOllamaTextModel, loaded.Vector.Embeddings.Model) + // The embedding-only model is never promoted to the sweep's chat model. + assert.False(loaded.People.Sweep.Enabled) + assert.NotContains(loaded.People.Sweep.Providers, setupOllamaProfile) + assert.Contains(output, "ollama pull gpt-oss-128k") +} + +func TestSetupProvidersDisclosureListsInferenceSources(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--dry-run") + require.NoError(err, output) + assert.Contains(output, "bounded evidence packets of conversation_text, meeting_text, document_text for tracked people") +} + +func TestSetupProvidersWithoutProvidersReportsEveryLaneOff(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + before := fixture.readConfig(t) + + output, err := fixture.run(t, "providers", "--json") + require.NoError(err, output) + assert.Equal(before, fixture.readConfig(t)) + + var result setupProvidersOutput + require.NoError(json.Unmarshal([]byte(output), &result)) + assert.False(result.Applied) + for _, lane := range result.Plan { + assert.Equal(planActionSkip, lane.Action, lane.Lane) + } + text := findLane(t, result.Report, laneTextSearch) + assert.Equal(laneStateOff, text.State) + assert.Contains(text.Reason, setupVoyageKeyEnv) +} + +func TestSetupProvidersDryRunWritesNothing(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + before := fixture.readConfig(t) + + output, err := fixture.run(t, "providers", "--dry-run") + require.NoError(err, output) + assert.Equal(before, fixture.readConfig(t)) + assert.Contains(output, "Dry run: nothing written.") + assert.Contains(output, "Voyage AI ("+setupVoyageEndpoint+") receives:") + assert.Contains(output, "message, chat, and meeting text") +} + +func TestSetupProvidersRequiresConsentWithoutTerminal(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + before := fixture.readConfig(t) + + _, err := fixture.run(t, "providers") + require.ErrorContains(t, err, "--yes") + require.ErrorContains(t, err, gateVoyage) + assert.Equal(before, fixture.readConfig(t)) +} + +func TestSetupProvidersDeclinedProviderWritesNothingForIt(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + fixture.tty = true + // Decline Voyage, accept Mistral. + fixture.input = strings.NewReader("n\ny\n") + + output, err := fixture.run(t, "providers") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.False(loaded.Vector.Enabled) + assert.False(loaded.Vector.People.Enabled) + assert.True(loaded.Attachments.Documents.Enabled) + // Document vectors need the declined text lane, so they were not enabled + // even though the Mistral gate was accepted. + assert.False(loaded.Attachments.Documents.Index.Embeddings.Enabled) + assert.Contains(output, "Enable the voyage lanes? [y/N]:") + assert.Contains(output, "Enable the mistral lanes? [y/N]:") +} + +func TestSetupProvidersKeepsConfiguredTextLane(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` +[vector] +enabled = true + +[vector.embeddings] +endpoint = "https://api.openai.com/v1" +api_key_env = "OPENAI_API_KEY" +model = "text-embedding-3-large" +dimension = 3072 +`) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + + loaded := fixture.load(t) + assert.Equal("text-embedding-3-large", loaded.Vector.Embeddings.Model) + assert.Equal(3072, loaded.Vector.Embeddings.Dimension) + assert.Equal(vector.APIFormatOpenAI, loaded.Vector.Embeddings.EffectiveAPIFormat()) + // The unset lanes still gained defaults. + assert.True(loaded.Vector.People.Enabled) + assert.True(loaded.People.Sweep.Enabled) + assert.Contains(output, "embeddings build --full-rebuild") +} + +func TestSetupProvidersRefusesConfiguredRemote(t *testing.T) { + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + deps := fixture.deps(t) + deps.remoteConfigured = func() bool { return true } + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newSetupProvidersCommand(deps)) + root.SetArgs([]string{"providers", "--yes"}) + root.SetOut(io.Discard) + root.SetErr(io.Discard) + + err := root.ExecuteContext(t.Context()) + require.ErrorContains(t, err, "remote daemon") + assert.Equal(t, strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir)), fixture.readConfig(t)) +} + +func TestSetupStatusReportsPendingLanesForPresentKeys(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + + output, err := fixture.run(t, "status", "--json") + require.NoError(err, output) + var report laneReport + require.NoError(json.Unmarshal([]byte(output), &report)) + + text := findLane(t, report, laneTextSearch) + assert.Equal(laneStatePending, text.State) + assert.Equal([]string{"msgvault setup providers"}, text.Next) + visual := findLane(t, report, laneVisualSearch) + assert.Equal(laneStatePending, visual.State) + documents := findLane(t, report, laneDocuments) + assert.Equal(laneStateOff, documents.State) + assert.Contains(documents.Reason, "MISTRAL_API_KEY") + activity := findLane(t, report, laneActivity) + assert.Equal(laneStateOn, activity.State) + assert.Equal("cron 17 * * * *", activity.Schedule) + media := findLane(t, report, laneMediaPolicy) + assert.Contains(media.Reason, "beeper scope all") + assert.Equal([]string{"search_people", "get_person_notes", "get_person_relationship", "search_person_files"}, report.MCPTools) + + human, err := fixture.run(t, "status") + require.NoError(err, human) + assert.Contains(human, "LANE") + assert.Contains(human, "Text search (messages, chats, meetings)") + assert.Contains(human, "MCP tools live with this configuration") +} + +func findLane(t *testing.T, report laneReport, lane string) laneStatus { + t.Helper() + for _, item := range report.Lanes { + if item.Lane == lane { + return item + } + } + require.Failf(t, "lane missing", "lane %q not in report", lane) + return laneStatus{} +} diff --git a/cmd/msgvault/cmd/setup_status.go b/cmd/msgvault/cmd/setup_status.go new file mode 100644 index 000000000..11ffbd6e3 --- /dev/null +++ b/cmd/msgvault/cmd/setup_status.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "errors" + "os" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" +) + +// setupStatusDeps isolates the report from the process so tests can drive it +// from a temp config and a fixed environment. +type setupStatusDeps struct { + config func() *config.Config + environment func(*cobra.Command, *config.Config) setupEnvironment +} + +func defaultSetupStatusDeps() setupStatusDeps { + return setupStatusDeps{ + config: func() *config.Config { return cfg }, + environment: func(command *cobra.Command, loaded *config.Config) setupEnvironment { + return setupEnvironment{ + lookupEnv: os.LookupEnv, + fileExists: defaultFileExists, + consent: readSetupConsentState(command.Context(), loaded), + } + }, + } +} + +func newSetupStatusCommand(deps setupStatusDeps) *cobra.Command { + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Report which retrieval and people lanes are on, with provider, model, and the next step", + Long: `Report every optional lane: text search, semantic people search, visual +attachment search, document extraction and vectors, the people sweep, the +activity projection, and the chat media policy. For each lane the report +names the provider and model, whether it is on, why it is off, whether its +consent is recorded, and the exact command that turns it on. + +The report reads config.toml, the process environment, and the local +archive's consent records. It never contacts a provider.`, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + loaded := deps.config() + if loaded == nil { + return errors.New("configuration is unavailable") + } + report := buildLaneReport(loaded, deps.environment(command, loaded)) + return writeLaneReport(command.OutOrStdout(), report, jsonOutput) + }, + } + command.Flags().BoolVar(&jsonOutput, flagJSON, false, "Output structured JSON") + return command +} diff --git a/docs/changelog.md b/docs/changelog.md index fcb2b1cd8..e81f029f4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -73,6 +73,21 @@ All notable changes to msgvault, grouped by release. criteria, with optional exact-source narrowing and `--dry-run` support for reviewing the match count first. +- `msgvault setup providers` turns on the retrieval and people lanes the + available API keys support, with recommended defaults: a Voyage key + configures contextual text search (`voyage-context-4`), semantic people + search, and the visual lane once its probe manifest exists; a Mistral key + configures document extraction and, with a text lane, document vectors; an + OpenAI key onboards the people sweep on `gpt-5.6-luna` (and the + OpenAI-compatible text lane when no Voyage key is present); with no hosted + key a reachable local Ollama server is used. Setup asks once per hosted + provider, never turns a hosted lane on from a key alone, leaves configured + lanes untouched, and prints the next commands. `msgvault setup status` + reports every lane with provider, model, consent state, schedule, and the + reason it is off. The `api_format`, `[vector.people]`, + `[vector.multimodal]`, and `[activity]` sections are now documented, and a + Recommended Configuration page lists the file setup writes. + - Starting in v0.20.0, remote deletion remains permanently opt-in. The invoking CLI can grant durable consent with `[deletion] remote_enabled = true`; `MSGVAULT_ENABLE_REMOTE_DELETE=1` diff --git a/docs/cli-reference.md b/docs/cli-reference.md index c1ecd8fd1..54dadf6f0 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1905,6 +1905,51 @@ If configured for a remote server, this command generates `/nas-b The wizard also stores remote URL/API key in `remote` config block so `export-token` can use it without extra flags. +### setup providers + +Turn on the retrieval and people lanes the available API keys support, with +recommended defaults. Reads `VOYAGE_API_KEY`, `MISTRAL_API_KEY`, and +`OPENAI_API_KEY` (and probes a local Ollama server at `[chat].server` when no +hosted key is present), prints a plan, asks once per hosted provider, writes +the recommended sections to `config.toml`, onboards the people-sweep +provider through the same check and consent gates as `person provider`, and +prints the lane report with the next commands. Lanes that are already +configured are left alone. See +[Recommended Configuration](/usage/recommended-configuration/). + +```bash +msgvault setup providers --dry-run +msgvault setup providers +msgvault setup providers --yes --document-retention zdr --document-training opted-out +``` + +| Flag | Default | Description | +|---|---|---| +| `--yes` | `false` | Accept every provider disclosure without prompting (required when stdin is not a terminal) | +| `--dry-run` | `false` | Print the plan, the disclosures, and the current lane report without writing | +| `--document-retention` | `standard` | Mistral retention posture to record: `standard` or `zdr` | +| `--document-training` | `default-opt-out` | Mistral training posture to record: `default-opt-out` or `opted-out` | +| `--retention-posture` | `provider-declared` | Retention assertion recorded for embedding and inference providers | +| `--training-posture` | `provider-declared` | Training assertion recorded for embedding and inference providers | +| `--json` | `false` | Output the plan, applied flag, follow-ups, and report as JSON | + +The command edits the config file on this machine and refuses to run against +a configured remote daemon; run it on the daemon host or pass `--local`. + +### setup status + +Report every lane (text search, semantic people search, visual attachments, +documents, document vectors, people sweep, activity projection, media +policy): state, provider, model, recorded consent, schedule, the reason a +lane is off, and the command that turns it on. Reads `config.toml`, the +environment, and the local archive's consent records; never contacts a +provider. + +```bash +msgvault setup status +msgvault setup status --json +``` + --- ## show-message diff --git a/docs/configuration.md b/docs/configuration.md index cc612dfba..addb2e24f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -340,6 +340,12 @@ client_secrets = 'C:\Users\you\Downloads\client_secret.json' ## Sections +`msgvault setup providers` writes recommended values for the `[vector]`, +`[attachments.documents]`, and `[people.sweep]` sections from the API keys in +your environment, and `msgvault setup status` reports every lane with its +provider, model, consent state, and next step. The values it chooses are +listed in [Recommended Configuration](/usage/recommended-configuration/). + ### `[data]` | Key | Default | Description | @@ -951,6 +957,7 @@ External OpenAI-compatible embedding endpoint used to convert message text into | Key | Default | Description | |---|---|---| +| `api_format` | `openai` | Request contract: `openai` (OpenAI-compatible `/embeddings`, one vector per message chunk) or `voyage-contextual` (Voyage `/contextualizedembeddings`; pins `model = "voyage-context-4"` and embeds chat conversation windows and turn-aware meeting chunks as contextual documents). | | `endpoint` | (required) | HTTP(S) base URL for an OpenAI-compatible embeddings API. msgvault appends `/embeddings` (for example, set `http://localhost:11434/v1`, not `.../embeddings`). | | `model` | (required) | Model name to pass in each request (e.g., `nomic-embed-text`). | | `dimension` | (required) | Vector dimension. Must match the model's output dimension. | @@ -1048,6 +1055,63 @@ Optional background scheduling for the embed worker inside `msgvault serve`. Emp | `cron` | — | 5-field cron expression. Empty string disables the standalone cron. | | `run_after_sync` | `false` | When `true`, an embed pass runs after every successful scheduled sync. | +`msgvault setup providers` sets `run_after_sync = true` and `cron = "*/15 * * * *"` when it enables a text lane. See [Recommended Configuration](/usage/recommended-configuration/). + +#### `[vector.people]` + +Semantic people search: one curated document per durable person, built from +searchable non-sensitive attributes, embedded into the text-search generation. +Requires `[vector] enabled = true` and a separate consent +(`msgvault person provider consent --semantic-embeddings --yes`). + +| Key | Default | Description | +|---|---|---| +| `enabled` | `false` | Embed curated person documents and serve `msgvault person search`. | +| `retention_posture` | — | Your assertion about the embedding provider's retention; must be explicit (not `unknown`). | +| `training_posture` | — | Your assertion about the embedding provider's training use; must be explicit. | + +#### `[vector.multimodal]` + +Independently consented visual attachment lane over Voyage. Every value has a +default except the probe manifest; uploads fail closed without it, and a +daemon started with `enabled = true` and no manifest refuses every vector +lane until one exists. + +| Key | Default | Description | +|---|---|---| +| `enabled` | `false` | Turn on the visual lane. | +| `provider` | `voyage` | Only legal value. | +| `endpoint` | `https://api.voyageai.com/v1` | Pinned provider root; other origins are refused. | +| `api_key_env` | `VOYAGE_API_KEY` | Environment variable holding the key. A key alone enables nothing. | +| `model` | `voyage-multimodal-3.5` | Pinned model. | +| `dimension` | `1024` | Pinned dimension. | +| `capabilities_file` | — | Manifest written by `msgvault multimodal probe --seeds --out --yes`. | +| `max_context_chars` | `4000` | Owning-message text sent with each attachment. | +| `include_images` | `true` | Embed still images (JPEG, PNG, WebP). | +| `include_animated_gifs` | `false` | Embed animated GIFs; requires `include_images` and a manifest that authorized them. | +| `include_video` | `true` | Embed direct-input MP4 video. | +| `allow_image_queries` | `true` | Allow `multimodal search --image`. | + +`[vector.multimodal.scope]` accepts the same `message_types` and `accounts` +keys as `[vector.embed.scope]`; `[vector.multimodal.schedule]` accepts the +same `cron` and `run_after_sync` keys as `[vector.embed.schedule]`. Consent is +recorded per generation by `msgvault multimodal build --yes`. + +### `[activity]` + +Dated activity projection and per-person contact state (first and last +contact, inbound/outbound, interaction count, inferred channel). It is the +deterministic source of "when did we last talk" for every person and runs +hourly by default inside `msgvault serve`. `msgvault activity build` runs it +by hand; `--backstop` rescans the whole archive. + +| Key | Default | Description | +|---|---|---| +| `schedule` | `17 * * * *` | 5-field cron used by `msgvault serve`. Empty disables the scheduled job. | +| `timezone` | `UTC` | IANA zone name for day bucketing. `Local` is rejected because the projection keys replay on the persisted zone name. | +| `max_direct_counterparts` | `25` | Largest conversation still projected as direct activity between its participants. | +| `batch_size` | `500` | Messages per projection batch. | + ## Overriding the Home Directory By default, msgvault stores everything under `~/.msgvault` (macOS/Linux) or `C:\Users\\.msgvault` (Windows). To use a different location, you have two options: diff --git a/docs/setup.md b/docs/setup.md index 80a01aa49..96d7e3b8c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -308,6 +308,23 @@ msgvault stats See [Web UI](/docs/web-ui/), [Searching](/docs/usage/searching/), and [Interactive TUI](/docs/usage/tui/) for more. +## Optional: Turn On Search and People Lanes + +Semantic search, visual and document attachment search, and the people sweep +are opt-in. Put the API keys you have in the environment and let setup choose +the rest: + +```bash +export VOYAGE_API_KEY="..." # text, people, and visual search +export MISTRAL_API_KEY="..." # document attachments +export OPENAI_API_KEY="..." # people sweep (and text search without a Voyage key) +msgvault setup providers # one consent per provider, then config.toml is written +msgvault setup status # what is on, what is off, and why +``` + +See [Recommended Configuration](/usage/recommended-configuration/) for the +values it writes and the probe steps the hosted lanes still need. + ## Optional: Sync Google Calendar To archive Calendar events alongside email, authorize Calendar access and run a diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md new file mode 100644 index 000000000..18a4339b9 --- /dev/null +++ b/docs/usage/recommended-configuration.md @@ -0,0 +1,211 @@ +--- +last_edited: "2026-09-03" +title: Recommended Configuration +description: The config.toml that `msgvault setup providers` writes from the API keys you have, section by section. +--- + +Msgvault has four retrieval lanes (message text, curated people, visual +attachments, document attachments), a people sweep that keeps profiles +current, a scheduled activity projection, and a media policy for chat +sources. Each shipped as opt-in with its own table, key variable, consent +step, and build command. `msgvault setup providers` chooses recommended +values for all of them from the keys in your environment and turns on the +lanes those keys support, one explicit consent per hosted provider. + +```bash +export VOYAGE_API_KEY="..." # text, people, and visual search +export MISTRAL_API_KEY="..." # document attachments +export OPENAI_API_KEY="..." # people sweep (and text search when no Voyage key) + +msgvault setup providers --dry-run # show the plan and each provider disclosure +msgvault setup providers # answer once per provider, write config.toml +msgvault setup status # what is on, what is off, and why +``` + +Hosted lanes never turn on from a key alone. Setup asks before it writes, +records the postures you assert, runs the people-provider check and consent +through the same gates the `person provider` commands enforce, and prints the +exact commands that finish the lanes it cannot complete on its own (the two +provider probes need private synthetic seed files). Re-running setup after +adding a key upgrades only the lanes that are still unset; a configured lane +keeps its model, because switching the embedding policy invalidates the +index and is your call. + +Every value below is settable per lane exactly as before. This page only +describes what happens when nothing is set. + +## What each key turns on + +| Key present | Lanes | Model | Notes | +|---|---|---|---| +| `VOYAGE_API_KEY` | text search, semantic people search, visual attachments (after the probe) | `voyage-context-4` (1024), `voyage-multimodal-3.5` (1024) | Chats embed as conversation windows and meetings as turn-aware chunks; email rides the same generation. | +| `MISTRAL_API_KEY` | document extraction and lexical search; document vectors when a text lane is on | `mistral-ocr-4-0`, EU region | Uploads are manual-only and need the probe manifest plus `documents consent-mistral --yes`. | +| `OPENAI_API_KEY` | people sweep; text search only when no Voyage key | `gpt-5.6-luna` at `medium` reasoning; `text-embedding-3-small` (1536) | The OpenAI text path gives per-message vectors: no conversation-window context and no visual lane, both are Voyage-only endpoints. | +| none | local Ollama at `[chat].server` when reachable | `nomic-embed-text` (768); the `[chat].model` for the sweep | Text stays on your machine. Setup skips a lane the server cannot serve and says why. | + +## The file setup writes + +With a Voyage key, a Mistral key, and an OpenAI key present, setup writes +the sections below into an otherwise empty `config.toml`. Comments and +sections you already have are preserved. + +```toml +[vector] +enabled = true +backend = "sqlite-vec" # "pgvector" when [data].database_url is PostgreSQL + +[vector.embeddings] +api_format = "voyage-contextual" # conversation windows and turn-aware meeting chunks +endpoint = "https://api.voyageai.com/v1" +api_key_env = "VOYAGE_API_KEY" +model = "voyage-context-4" +dimension = 1024 + +[vector.embed.schedule] +run_after_sync = true # embed after every successful scheduled sync +cron = "*/15 * * * *" # and catch up chat sources that do not trigger a post-sync pass + +[vector.people] +enabled = true # one curated, non-sensitive document per person +retention_posture = "provider-declared" +training_posture = "provider-declared" + +[vector.multimodal.schedule] +run_after_sync = true +cron = "*/15 * * * *" +# [vector.multimodal] enabled = true and capabilities_file are written once the +# probe manifest exists at /voyage-capabilities.json. + +[attachments.documents] +enabled = true +retention_posture = "standard" # or "zdr"; --document-retention +training_posture = "default-opt-out" # or "opted-out"; --document-training + +[attachments.documents.index.embeddings] +enabled = true # document chunks use the text-search profile after `documents vectors consent` + +[people.sweep] +enabled = true +provider = "openai" + +[people.sweep.providers.openai] +protocol = "openai_chat" +endpoint = "https://api.openai.com/v1" +model = "gpt-5.6-luna" +auth = "bearer" +credential = "env" +credential_env = "OPENAI_API_KEY" +output_mode = "native_json_schema" +token_limit_parameter = "max_completion_tokens" +reasoning_effort = "medium" +retention_posture = "provider-declared" +training_posture = "provider-declared" +allowed_sources = ["conversation_text", "meeting_text", "document_text"] +source_since = "2025-01-01" # January 1 of last year +allow_sensitive = true +request_timeout = "1m0s" +``` + +### `[vector]` and `[vector.embeddings]` + +The text lane. `api_format = "voyage-contextual"` pins `voyage-context-4` +and sends each chat conversation window and each meeting as one contextual +request, so a message is embedded with its neighbors. The OpenAI-compatible +format (`api_format = "openai"`) embeds each message on its own. Message +text leaves the machine either way; setup states that before it asks. + +`run_after_sync` covers Gmail, IMAP, Teams, and Discord syncs. The cron +covers Slack, Beeper, calendar, and meeting sources, which do not trigger a +post-sync embed. See [Vector Search](/usage/vector-search/). + +### `[vector.people]` + +Semantic people search embeds one curated document per durable person +(searchable, non-sensitive attributes only) into the same generation, so a +query like "a finance contact in Berlin" returns the person. The postures are +your assertion about the embedding provider; setup records +`provider-declared` unless you pass `--retention-posture` and +`--training-posture`. Consent is a separate step: +`msgvault person provider consent --semantic-embeddings --yes`. + +### `[vector.multimodal]` + +The visual lane needs a capability manifest from an authenticated probe of +Voyage with private synthetic fixtures, and the probe needs four seed files +you supply (a WebP and an MP4, each with a contrasting variant). Enabling the +lane without the manifest makes the daemon refuse every vector lane, so setup +writes only the schedule until the manifest exists: + +```bash +msgvault multimodal probe --seeds --out ~/.msgvault/voyage-capabilities.json --yes +msgvault setup providers # now enables [vector.multimodal] with that manifest +msgvault daemon restart +msgvault multimodal build --yes # consent to exactly that capability profile +``` + +### `[attachments.documents]` + +Mistral is the only document provider and receives the complete original +bytes of standalone document attachments. Setup records the least-asserting +legal postures (`standard` retention, `default-opt-out` training) unless you +pass `--document-retention zdr` or `--document-training opted-out`; use the +values your account actually has. Uploads stay manual: build the fixture +matrix, probe, consent, then build. See +[Document Attachment Indexing](/usage/document-indexing/). + +```bash +msgvault documents probe-mistral --fixtures > ~/.msgvault/mistral-capabilities.json +msgvault documents consent-mistral --capabilities ~/.msgvault/mistral-capabilities.json --yes +msgvault documents build --capabilities ~/.msgvault/mistral-capabilities.json --yes +msgvault documents vectors consent --yes # when document vectors are enabled +``` + +### `[people.sweep]` + +The sweep keeps curated attributes current from the archive for people you +track (`msgvault person track `). Deterministic contact state +(last contacted, cadence, inferred channel) refreshes hourly for everyone +through `[activity]` and needs no model. Setup onboards the `openai` profile +through `person provider add` (a synthetic check request is sent), records +consent, and selects it; the daily schedule is the `[people.sweep]` default. +`allow_sensitive = true` is required for real sweeps because every evidence +packet is marked sensitive. The Codex app-server adapter is release-gated and +cannot be the default. With no OpenAI key, setup offers a loopback Ollama +profile on `[chat].model`. + +### `[activity]` + +On by default (`17 * * * *`, UTC). It projects archived messages into dated +per-person contact state. Nothing to configure; see +[Configuration](/configuration/#activity). + +### Media policy + +Chat sources cap collection by conversation size: media from rooms above 20 +participants is skipped with a typed `participant_threshold` marker, direct +and small-group media is kept. Set `media_max_participants = 0` on a source +to lift the cap. See the `[beeper]`, `[slack]`, `[discord]`, and `[teams]` sections of [Configuration](/configuration/#beeper). + +## What the MCP server answers with these defaults + +The people tools (`search_people`, `get_person_notes`, +`get_person_relationship`, `search_person_files`) read local derived state +and are on regardless of provider keys. The text lane adds +`semantic_search_messages` and `find_similar_messages`; the visual lane adds +`search_visual_attachments`; the document lane adds +`search_document_attachments`. `msgvault setup status` prints the live list. + +## Reading the status report + +```text +LANE STATE PROVIDER MODEL CONSENT SCHEDULE +Text search (messages, chats, meetings) on voyage voyage-context-4 - cron */15 * * * *, after each scheduled sync +Semantic people search on voyage voyage-context-4 missing - +Visual attachment search pending - - - - +Document attachments (...) on mistral mistral-ocr-4-0 missing - +``` + +`pending` means the lane is configured or the key is present but an +operator step remains; the `next` line under the table names it. `unknown` +consent means the archive could not be read (for example, the database does +not exist yet). Use `--json` for scripting. diff --git a/docs/zensical.toml b/docs/zensical.toml index 93f4c2422..0fda4d5f4 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -25,6 +25,7 @@ nav = [ {"IMAP Folder Sync" = "usage/imap.md"}, {"Searching" = "usage/searching.md"}, {"Vector Search" = "usage/vector-search.md"}, + {"Recommended Configuration" = "usage/recommended-configuration.md"}, {"Document Attachment Indexing" = "usage/document-indexing.md"}, {"Importing Local Email" = "usage/importing.md"}, {"Text Messages" = "usage/text-messages.md"}, From 92f271011b3781ff2aa5dd079b65a6894a75c7da Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 4 Sep 2026 19:34:48 -0500 Subject: [PATCH 02/10] fix(setup): restore configuration when provider onboarding fails A failed people-provider check, consent, or selection left other lanes configured, so a retry treated the incomplete setup as finished. Restore all setup config edits on failure, including a newly created file, using the published file identity to preserve concurrent edits. Recompute dependent lanes after a provider is declined. Declining Mistral must also disable document vectors and remove document_text from the people-sweep profile and its subsequent disclosure. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/setup_providers.go | 114 ++++++++++++++++++--- cmd/msgvault/cmd/setup_providers_test.go | 124 +++++++++++++++++++++++ 2 files changed, 222 insertions(+), 16 deletions(-) diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 5581443eb..e6bf94037 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "slices" "sort" "strings" "text/tabwriter" @@ -63,18 +64,19 @@ type ollamaProbeResult struct { // environment, the config file, the archive, the daemon, and the people // provider onboarding machinery are all injectable. type setupProvidersDeps struct { - lookupEnv func(string) (string, bool) - fileExists func(string) bool - readConfigFile func() (config.ConfigFile, error) - editConfigTables func(string, []config.TableEdit) (config.ConfigFile, error) - loadConfig func(config.ConfigFile) (*config.Config, error) - remoteConfigured func() bool - isTerminal func(*cobra.Command) bool - probeOllama func(context.Context, string) ollamaProbeResult - consentState func(context.Context, *config.Config) *setupConsentState - daemonAlive func(context.Context, *config.Config) bool - personProvider func() personProviderCommandDeps - now func() time.Time + lookupEnv func(string) (string, bool) + fileExists func(string) bool + readConfigFile func() (config.ConfigFile, error) + editConfigTables func(string, []config.TableEdit) (config.ConfigFile, error) + restoreConfigFile func(config.ConfigFile, config.ConfigFile) (config.ConfigFile, error) + loadConfig func(config.ConfigFile) (*config.Config, error) + remoteConfigured func() bool + isTerminal func(*cobra.Command) bool + probeOllama func(context.Context, string) ollamaProbeResult + consentState func(context.Context, *config.Config) *setupConsentState + daemonAlive func(context.Context, *config.Config) bool + personProvider func() personProviderCommandDeps + now func() time.Time } func defaultSetupProvidersDeps() setupProvidersDeps { @@ -93,6 +95,9 @@ func defaultSetupProvidersDeps() setupProvidersDeps { } return config.EditConfigTables(cfg.ConfigFilePath(), ifMatch, edits) }, + restoreConfigFile: func(published, before config.ConfigFile) (config.ConfigFile, error) { + return config.RestoreConfigFile(before.LogicalPath, published, before) + }, loadConfig: func(snapshot config.ConfigFile) (*config.Config, error) { if cfg == nil { return nil, errors.New("configuration is unavailable") @@ -308,6 +313,24 @@ func (p *setupProvidersPlan) declineGate(gate string) { } } +// Refresh only still-approved dependent lanes. A declined lane must not be +// proposed again, and subsequent disclosures must describe the reduced plan. +func (p *setupProvidersPlan) refreshDependencies(loaded *config.Config, detection setupDetection, options setupProvidersOptions, now time.Time) { + for i, lane := range p.Lanes { + if lane.Action != planActionEnable && lane.Action != planActionOnboard { + continue + } + switch lane.Lane { + case lanePersonSearch: + p.Lanes[i] = planPersonSearch(loaded, p, options) + case laneDocumentVectors: + p.Lanes[i] = planDocumentVectors(loaded, p) + case lanePeopleInference: + p.Lanes[i], p.inference = planPeopleInference(loaded, detection, p, options, now) + } + } +} + // mergedEdits collapses the plan into one table edit per path so a single // ETag-guarded write publishes every lane together. func (p *setupProvidersPlan) mergedEdits() []config.TableEdit { @@ -840,6 +863,9 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options } reader := bufio.NewReader(command.InOrStdin()) for _, gate := range gates { + if !slices.Contains(plan.gates(), gate) { + continue + } _, _ = fmt.Fprintln(out, gateDisclosure(gate, &plan)) _, _ = fmt.Fprintf(out, "Enable the %s lanes? [y/N]: ", gate) answer, readErr := reader.ReadString('\n') @@ -849,6 +875,7 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options if !isYesAnswer(strings.ToLower(strings.TrimSpace(answer))) { declined = append(declined, gate) plan.declineGate(gate) + plan.refreshDependencies(loaded, detection, options, now) } _, _ = fmt.Fprintln(out) } @@ -859,21 +886,76 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options } } + if err := applySetupProvidersPlan(command, deps, before, &plan, options.jsonOutput); err != nil { + return err + } + return writeSetupProvidersResult(command, deps, nil, &plan, options, plan.writes(), declined) +} + +// The people-provider commands need a published profile for daemon-owned +// checks and consent. Keep their existing gates and restore the entire setup +// configuration if any step fails. Retain each publication's identity so a +// concurrent edit is never adopted as our rollback target. +func applySetupProvidersPlan(command *cobra.Command, deps setupProvidersDeps, before config.ConfigFile, plan *setupProvidersPlan, quiet bool) (retErr error) { edits := plan.mergedEdits() + if len(edits) == 0 && plan.inference == nil { + return nil + } + if deps.restoreConfigFile == nil { + return errors.New("setup providers config rollback is unavailable") + } + current := before + changed := false + defer func() { + if retErr != nil && changed && current.Exists { + if _, err := deps.restoreConfigFile(current, before); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("restore setup config: %w", err)) + } + } + }() + record := func(snapshot config.ConfigFile, err error) (config.ConfigFile, error) { + if err == nil || (errors.Is(err, config.ErrConfigChanged) && snapshot.Exists) { + current = snapshot + changed = true + } + return snapshot, err + } + edit := func(etag string, edits []config.TableEdit) (config.ConfigFile, error) { + if etag != current.ETag { + return config.ConfigFile{}, config.ErrConfigConflict + } + return record(deps.editConfigTables(etag, edits)) + } if len(edits) > 0 { if err := config.ValidateConfigTableEdits(before, edits); err != nil { return fmt.Errorf("planned config changes are invalid: %w", err) } - if _, err := deps.editConfigTables(before.ETag, edits); err != nil { + if _, err := edit(before.ETag, edits); err != nil { return fmt.Errorf("write config: %w", err) } } - if plan.inference != nil && deps.personProvider != nil { - if err := onboardSetupInferenceProfile(command, deps.personProvider(), *plan.inference, options.jsonOutput); err != nil { + if plan.inference != nil { + if deps.personProvider == nil { + return errors.New("people provider onboarding is unavailable") + } + provider := deps.personProvider() + provider.readConfigFile = func() (config.ConfigFile, error) { + snapshot, err := deps.readConfigFile() + if err == nil && (snapshot.Exists != current.Exists || + (current.Exists && !config.SameConfigFileVersion(snapshot, current))) { + return config.ConfigFile{}, config.ErrConfigConflict + } + return snapshot, err + } + provider.editConfigTables = edit + provider.restoreConfigFile = func(published, previous config.ConfigFile) (config.ConfigFile, error) { + return record(deps.restoreConfigFile(published, previous)) + } + if err := onboardSetupInferenceProfile(command, provider, *plan.inference, quiet); err != nil { return err } } - return writeSetupProvidersResult(command, deps, nil, &plan, options, plan.writes(), declined) + return nil } // writeSetupProvidersResult reloads the config (unless the caller passes the diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 8e3ff1133..17d623f8f 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "os" "path/filepath" @@ -112,6 +114,9 @@ func (f *setupProvidersFixture) deps(t *testing.T) setupProvidersDeps { editConfigTables: func(etag string, edits []config.TableEdit) (config.ConfigFile, error) { return config.EditConfigTables(f.path, etag, edits) }, + restoreConfigFile: func(published, before config.ConfigFile) (config.ConfigFile, error) { + return config.RestoreConfigFile(f.path, published, before) + }, loadConfig: func(snapshot config.ConfigFile) (*config.Config, error) { return loadSetupConfig(snapshot, f.dir) }, @@ -422,6 +427,125 @@ func TestSetupProvidersDisclosureListsInferenceSources(t *testing.T) { assert.Contains(output, "bounded evidence packets of conversation_text, meeting_text, document_text for tracked people") } +func TestSetupProvidersDeclinedDocumentsUpdateDependentLanes(t *testing.T) { + for _, local := range []bool{false, true} { + t.Run(fmt.Sprint("local=", local), func(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + fixture.tty = true + profileName := setupInferenceProfile + if local { + fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} + fixture.input = strings.NewReader("n\n") + profileName = setupOllamaProfile + } else { + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + fixture.input = strings.NewReader("n\ny\n") + } + + output, err := fixture.run(t, "providers") + require.NoError(t, err, output) + loaded := fixture.load(t) + assert.False(loaded.Attachments.Documents.Enabled) + assert.False(loaded.Attachments.Documents.Index.Embeddings.Enabled) + assert.ElementsMatch([]peoplesweep.SourceClass{peoplesweep.SourceConversationText, peoplesweep.SourceMeetingText}, + loaded.People.Sweep.Providers[profileName].AllowedSources) + assert.NotContains(output, "bounded evidence packets of conversation_text, meeting_text, document_text") + assert.NotContains(output, "msgvault documents vectors consent --yes") + }) + } +} + +func TestSetupProvidersFailureRestoresConfig(t *testing.T) { + for _, stage := range []string{"negotiate", "check", "consent", "selection"} { + for _, content := range []string{"", setupProvidersMinimalConfig} { + t.Run(fmt.Sprintf("%s/missing=%t", stage, content == ""), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, content) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + before := fixture.readConfig(t) + failure := errors.New("provider setup failed") + deps := fixture.deps(t) + deps.personProvider = func() personProviderCommandDeps { + provider := fixture.personProviderDeps(t) + switch stage { + case "negotiate": + provider.setup.negotiate = func(context.Context, peoplesweep.ProviderConfig, peoplesweep.Credential) (peoplesweep.NegotiatedCapabilities, error) { + return peoplesweep.NegotiatedCapabilities{}, failure + } + case "check": + fixture.checker.err = failure + case "consent": + openStore := provider.openStore + provider.openStore = func() (personProviderStore, func(), error) { + if fixture.checker.calls.Load() > 0 { + return nil, nil, failure + } + return openStore() + } + case "selection": + provider.openReadStore = func() (personProviderStore, func(), error) { + return nil, nil, failure + } + } + return provider + } + command := newSetupProvidersCommand(deps) + command.SetArgs([]string{"--yes"}) + command.SetOut(io.Discard) + command.SetErr(io.Discard) + + err := command.ExecuteContext(t.Context()) + require.ErrorIs(err, failure) + assert.Equal(before, fixture.readConfig(t)) + if content == "" { + _, err := os.Stat(fixture.path) + require.ErrorIs(err, os.ErrNotExist) + } + fixture.checker.err = nil + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + assert.True(fixture.load(t).People.Sweep.Enabled) + }) + } + } +} + +func TestSetupProvidersRollbackPreservesConcurrentConfig(t *testing.T) { + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + deps := fixture.deps(t) + failure := errors.New("provider check failed") + var concurrent config.ConfigFile + deps.personProvider = func() personProviderCommandDeps { + provider := fixture.personProviderDeps(t) + provider.newChecker = func(peoplesweep.Config, personProviderStore) (personProviderChecker, error) { + return callbackPersonProviderChecker(func(context.Context) (peoplesweep.StructuredResponse, error) { + before, err := config.ReadConfigFile(fixture.path) + require.NoError(t, err) + concurrent, err = config.EditConfigTables(fixture.path, before.ETag, []config.TableEdit{{ + Path: []string{"activity"}, Values: map[string]any{"schedule": "0 * * * *"}, + }}) + require.NoError(t, err) + return peoplesweep.StructuredResponse{}, failure + }), nil + } + return provider + } + command := newSetupProvidersCommand(deps) + command.SetArgs([]string{"--yes"}) + command.SetOut(io.Discard) + command.SetErr(io.Discard) + + err := command.ExecuteContext(t.Context()) + require.ErrorIs(t, err, failure) + require.ErrorIs(t, err, config.ErrConfigConflict) + assert.Equal(t, string(concurrent.Content), fixture.readConfig(t)) +} + func TestSetupProvidersWithoutProvidersReportsEveryLaneOff(t *testing.T) { assert := assert.New(t) require := require.New(t) From e8eb88576eba851ed680b06fd514a1fdabf80cad Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 4 Sep 2026 20:06:45 -0500 Subject: [PATCH 03/10] fix(setup): require explicit consent for sensitive inference Keep the people sweep pending until --allow-sensitive explicitly permits sensitive archive excerpts and personal inferences. Describe that policy in both human and JSON plans; --yes alone must not authorize it. Leave vector lanes pending when the binary lacks the database backend, and report missing credential environment variables for hosted lanes so the setup report does not imply that unusable providers are ready. Fix the new documentation links to use the /docs/ URL prefix required by the published site and its link checker. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/setup_lanes.go | 14 ++++ cmd/msgvault/cmd/setup_providers.go | 59 +++++++++++---- cmd/msgvault/cmd/setup_providers_test.go | 96 ++++++++++++++++++++++-- docs/changelog.md | 8 +- docs/cli-reference.md | 9 ++- docs/configuration.md | 4 +- docs/setup.md | 5 +- docs/usage/recommended-configuration.md | 32 +++++--- 8 files changed, 187 insertions(+), 40 deletions(-) diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index ae5828869..27fec566e 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -12,6 +12,7 @@ import ( "go.kenn.io/msgvault/internal/attachmentpolicy" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/peoplesweep" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" ) @@ -109,6 +110,13 @@ func (e setupEnvironment) hasEnv(name string) bool { return ok && strings.TrimSpace(value) != "" } +func (e setupEnvironment) reportMissingCredential(lane *laneStatus, name string) { + if name != "" && !e.hasEnv(name) { + lane.State = laneStatePending + lane.Reason += "; environment variable " + name + " is not set" + } +} + func (e setupEnvironment) exists(path string) bool { if e.fileExists == nil || path == "" { return false @@ -305,12 +313,14 @@ func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.State = laneStatePending lane.Reason = "capabilities_file is missing; the daemon refuses every vector lane until it exists" lane.Next = []string{visualProbeCommand(cfg)} + env.reportMissingCredential(&lane, multimodal.APIKeyEnv) return lane } lane.Reason = "eligible images and short videos are embedded with bounded message context" if lane.Consent != consentActive { lane.Next = []string{"msgvault multimodal build --yes"} } + env.reportMissingCredential(&lane, multimodal.APIKeyEnv) return lane } lane.State = laneStateOff @@ -357,6 +367,7 @@ func documentsLane(cfg *config.Config, env setupEnvironment) laneStatus { } } } + env.reportMissingCredential(&lane, documents.APIKeyEnv) return lane } lane.State = laneStateOff @@ -411,6 +422,9 @@ func peopleInferenceLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Next = []string{"msgvault person provider consent " + name + " --yes"} } lane.Next = append(lane.Next, "msgvault person track ") + if err == nil && provider.Auth != peoplesweep.AuthNone && provider.Credential == peoplesweep.CredentialEnv { + env.reportMissingCredential(&lane, provider.CredentialEnv) + } return lane } lane.State = laneStateOff diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index e6bf94037..581c6f7e4 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -23,6 +23,8 @@ import ( "go.kenn.io/msgvault/internal/documentindex" "go.kenn.io/msgvault/internal/peoplesweep" "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector/pgvector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" ) const ( @@ -48,6 +50,7 @@ type setupProvidersOptions struct { yes bool dryRun bool jsonOutput bool + allowSensitive bool documentRetention string documentTraining string retentionPosture string @@ -188,16 +191,17 @@ func (r ollamaProbeResult) hasModel(name string) bool { // the local Ollama server offers, which probe manifests are already written, // and which vector backend the archive selects. type setupDetection struct { - voyageKey bool - mistralKey bool - mistralKeyEnv string - openAIKey bool - ollama ollamaProbeResult - ollamaEndpoint string - ollamaLoopback bool - voyageManifest string - mistralManifest string - backend string + voyageKey bool + mistralKey bool + mistralKeyEnv string + openAIKey bool + ollama ollamaProbeResult + ollamaEndpoint string + ollamaLoopback bool + voyageManifest string + mistralManifest string + backend string + backendUnavailable string } func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setupProvidersDeps) setupDetection { @@ -211,6 +215,11 @@ func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setup detection.mistralKey = env.hasEnv(detection.mistralKeyEnv) if store.IsPostgresURL(loaded.DatabaseDSN()) { detection.backend = "pgvector" + if !pgvector.Available() { + detection.backendUnavailable = "pgvector support is not compiled in; rebuild with `go build -tags \"fts5 sqlite_vec pgvector\"`, then re-run setup" + } + } else if !sqlitevec.Available() { + detection.backendUnavailable = "sqlite-vec support is not compiled in; rebuild with `make build`, then re-run setup" } if path := loaded.Vector.Multimodal.CapabilitiesFile; path != "" && env.exists(path) { detection.voyageManifest = path @@ -441,6 +450,10 @@ func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePl lane.Reason = "configured but disabled; set [vector] enabled = true to turn it on" return lane } + if detection.backendUnavailable != "" { + lane.Action, lane.Reason = planActionPending, detection.backendUnavailable + return lane + } vectorEdit := config.TableEdit{Path: []string{tomlTableVector}, Values: map[string]any{"enabled": true, "backend": detection.backend}} switch { case detection.voyageKey: @@ -529,6 +542,8 @@ func planVisualSearch(loaded *config.Config, detection setupDetection) setupLane lane.Action = planActionSkip lane.Provider, lane.Model = "", "" lane.Reason = "needs " + setupVoyageKeyEnv + case detection.backendUnavailable != "": + lane.Action, lane.Reason = planActionPending, detection.backendUnavailable case detection.voyageManifest != "": lane.Action, lane.Gate = planActionEnable, gateVoyage lane.Reason = "probe manifest found at " + detection.voyageManifest @@ -627,6 +642,13 @@ func planPeopleInference( lane.Reason = "already enabled" return lane, nil } + if !options.allowSensitive && (detection.openAIKey || + (detection.ollama.Reachable && detection.ollamaLoopback && detection.ollama.hasModel(loaded.Chat.Model))) { + lane.Action = planActionPending + lane.Reason = "people sweep requires --allow-sensitive: sensitive archive excerpts may be sent to the selected inference provider and used to infer sensitive personal attributes" + lane.next = []string{"msgvault setup providers --allow-sensitive"} + return lane, nil + } sources := []string{string(peoplesweep.SourceConversationText), string(peoplesweep.SourceMeetingText)} if plan.laneOn(laneDocuments) { sources = append(sources, string(peoplesweep.SourceDocumentText)) @@ -635,7 +657,7 @@ func planPeopleInference( base := personProviderAddOptions{ custom: true, protocol: string(peoplesweep.ProtocolOpenAIChat), retentionPosture: options.retentionPosture, trainingPosture: options.trainingPosture, - allowedSources: sources, sourceSince: since, allowSensitive: true, + allowedSources: sources, sourceSince: since, allowSensitive: options.allowSensitive, requestTimeout: time.Minute, confirmed: true, } switch { @@ -650,7 +672,7 @@ func planPeopleInference( base.endpoint, base.model, base.auth = setupOpenAIEndpoint, setupInferenceModel, string(peoplesweep.AuthBearer) base.credentialEnv, base.reasoningEffort = setupOpenAIKeyEnv, setupInferenceReasoning lane.Action, lane.Provider, lane.Model, lane.Gate = planActionOnboard, setupInferenceProfile, setupInferenceModel, gateOpenAI - lane.Reason = fmt.Sprintf("openai_chat profile %q at %s reasoning; evidence from %s since %s; extraction runs for tracked people only", + lane.Reason = fmt.Sprintf("openai_chat profile %q at %s reasoning; sensitive archive excerpts from %s since %s may be sent to OpenAI and used to infer sensitive personal attributes; extraction runs for tracked people only", setupInferenceProfile, setupInferenceReasoning, strings.Join(sources, ", "), since) lane.next = []string{"msgvault person track "} return lane, &setupInferencePlan{name: setupInferenceProfile, options: base, gate: gateOpenAI} @@ -668,7 +690,7 @@ func planPeopleInference( model := loaded.Chat.Model base.endpoint, base.model, base.auth = detection.ollamaEndpoint, model, string(peoplesweep.AuthNone) lane.Action, lane.Provider, lane.Model = planActionOnboard, setupOllamaProfile, model - lane.Reason = "local Ollama server at " + detection.ollamaEndpoint + "; evidence stays on this machine" + lane.Reason = "local Ollama server at " + detection.ollamaEndpoint + "; sensitive archive excerpts may be used to infer sensitive personal attributes; evidence stays on this machine" lane.next = []string{"msgvault person track "} return lane, &setupInferencePlan{name: setupOllamaProfile, options: base} case detection.ollama.Reachable && !detection.ollamaLoopback: @@ -723,7 +745,8 @@ func gateDisclosure(gate string, plan *setupProvidersPlan) string { if plan.inference != nil && plan.inference.gate == gateOpenAI { lines = append(lines, " - bounded evidence packets of "+ strings.Join(plan.inference.options.allowedSources, ", ")+ - " for tracked people ("+setupInferenceModel+"); a synthetic check request is sent now") + " for tracked people ("+setupInferenceModel+"); a synthetic check request is sent now", + " - --allow-sensitive authorizes sending sensitive archive excerpts to OpenAI and inferring sensitive personal attributes") } } return strings.Join(lines, "\n") @@ -792,7 +815,11 @@ writes the recommended values to config.toml, runs the people-provider check and consent, and prints what is on, what is off, and why. Lanes that are already configured are left alone, so re-running after adding a key upgrades only that lane. Probe manifests are expected at -/` + setupVoyageManifestName + ` and /` + setupMistralManifestName + `.`, +/` + setupVoyageManifestName + ` and /` + setupMistralManifestName + `. + +The people sweep also requires --allow-sensitive: archive excerpts may contain +sensitive details and may be used to infer sensitive personal attributes. +--yes accepts provider prompts but does not grant this separate opt-in.`, Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { return runSetupProviders(command, deps, options) @@ -802,6 +829,8 @@ upgrades only that lane. Probe manifests are expected at flags.BoolVar(&options.yes, "yes", false, "Accept every provider disclosure without prompting") flags.BoolVar(&options.dryRun, "dry-run", false, "Print the plan and the current lane report without writing anything") flags.BoolVar(&options.jsonOutput, flagJSON, false, "Output structured JSON") + flags.BoolVar(&options.allowSensitive, "allow-sensitive", false, + "Allow the people sweep to send sensitive archive excerpts to its inference provider and infer sensitive personal attributes") flags.StringVar(&options.documentRetention, "document-retention", documentindex.RetentionStandard, "Mistral retention posture to record: standard or zdr") flags.StringVar(&options.documentTraining, "document-training", documentindex.TrainingDefaultOptOut, diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 17d623f8f..1a4e77f68 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -22,6 +22,7 @@ import ( "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/pgvector" ) const setupProvidersTestKey = "setup-providers-test-key" @@ -176,6 +177,84 @@ const setupProvidersMinimalConfig = `# operator comment survives setup data_dir = "{{DIR}}/data" ` +func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+`database_url = "postgres://localhost/setup_test"`) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + output, err := fixture.run(t, "providers", "--yes", "--json") + require.NoError(err, output) + loaded := fixture.load(t) + assert.Equal(pgvector.Available(), loaded.Vector.Enabled) + assert.Equal(pgvector.Available(), loaded.Vector.Multimodal.Enabled) + assert.Equal(pgvector.Available(), loaded.Vector.People.Enabled) + previous := cfg + cfg = loaded + t.Cleanup(func() { cfg = previous }) + require.NoError(precheckVectorFeatures(loaded.DatabaseDSN())) + if !pgvector.Available() { + var result setupProvidersOutput + require.NoError(json.Unmarshal([]byte(output), &result)) + assert.False(result.Applied) + assert.Contains(output, "rebuild") + } +} + +func TestSetupProvidersRequiresSensitiveOptIn(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + output, err := fixture.run(t, "providers", "--yes", "--json") + require.NoError(err, output) + assert.False(fixture.load(t).People.Sweep.Enabled) + assert.NotContains(fixture.load(t).People.Sweep.Providers, setupInferenceProfile) + assert.Zero(fixture.checker.calls.Load()) + assert.Contains(output, "--allow-sensitive") + + output, err = fixture.run(t, "providers", "--yes", "--allow-sensitive", "--json") + require.NoError(err, output) + assert.Contains(output, "sensitive archive excerpts") + assert.EqualValues(1, fixture.checker.calls.Load()) + profile, err := fixture.load(t).People.Sweep.Profile() + require.NoError(err) + consented, err := fixture.store.HasActivePersonInferenceConsent(t.Context(), profile.Fingerprint) + require.NoError(err) + assert.True(consented) + assert.True(profile.AllowSensitive) +} + +func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + for _, key := range []string{setupVoyageKeyEnv, "MISTRAL_API_KEY", setupOpenAIKeyEnv} { + fixture.env[key] = setupProvidersTestKey + } + fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + _, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + require.NoError(err) + for lane, key := range map[string]string{ + laneVisualSearch: setupVoyageKeyEnv, laneDocuments: "MISTRAL_API_KEY", lanePeopleInference: setupOpenAIKeyEnv, + } { + fixture.env[key] = " " + output, err := fixture.run(t, "status", "--json") + require.NoError(err, output) + var report laneReport + require.NoError(json.Unmarshal([]byte(output), &report)) + status := findLane(t, report, lane) + assert.Equal(laneStatePending, status.State) + assert.Contains(status.Reason, key) + assert.Contains(status.Reason, "not set") + fixture.env[key] = setupProvidersTestKey + output, err = fixture.run(t, "status", "--json") + require.NoError(err, output) + require.NoError(json.Unmarshal([]byte(output), &report)) + assert.Equal(laneStateOn, findLane(t, report, lane).State) + } +} + func TestSetupProvidersVoyageWritesRecommendedDefaults(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -291,7 +370,7 @@ func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--yes") + output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(err, output) loaded := fixture.load(t) @@ -348,7 +427,7 @@ func TestSetupProvidersLocalOllamaFallback(t *testing.T) { fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} - output, err := fixture.run(t, "providers") + output, err := fixture.run(t, "providers", "--allow-sensitive") require.NoError(err, output) loaded := fixture.load(t) @@ -422,9 +501,10 @@ func TestSetupProvidersDisclosureListsInferenceSources(t *testing.T) { fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--dry-run") + output, err := fixture.run(t, "providers", "--dry-run", "--allow-sensitive") require.NoError(err, output) assert.Contains(output, "bounded evidence packets of conversation_text, meeting_text, document_text for tracked people") + assert.Contains(output, "--allow-sensitive authorizes sending sensitive archive excerpts to OpenAI") } func TestSetupProvidersDeclinedDocumentsUpdateDependentLanes(t *testing.T) { @@ -444,7 +524,7 @@ func TestSetupProvidersDeclinedDocumentsUpdateDependentLanes(t *testing.T) { fixture.input = strings.NewReader("n\ny\n") } - output, err := fixture.run(t, "providers") + output, err := fixture.run(t, "providers", "--allow-sensitive") require.NoError(t, err, output) loaded := fixture.load(t) assert.False(loaded.Attachments.Documents.Enabled) @@ -494,7 +574,7 @@ func TestSetupProvidersFailureRestoresConfig(t *testing.T) { return provider } command := newSetupProvidersCommand(deps) - command.SetArgs([]string{"--yes"}) + command.SetArgs([]string{"--yes", "--allow-sensitive"}) command.SetOut(io.Discard) command.SetErr(io.Discard) @@ -506,7 +586,7 @@ func TestSetupProvidersFailureRestoresConfig(t *testing.T) { require.ErrorIs(err, os.ErrNotExist) } fixture.checker.err = nil - output, err := fixture.run(t, "providers", "--yes") + output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(err, output) assert.True(fixture.load(t).People.Sweep.Enabled) }) @@ -536,7 +616,7 @@ func TestSetupProvidersRollbackPreservesConcurrentConfig(t *testing.T) { return provider } command := newSetupProvidersCommand(deps) - command.SetArgs([]string{"--yes"}) + command.SetArgs([]string{"--yes", "--allow-sensitive"}) command.SetOut(io.Discard) command.SetErr(io.Discard) @@ -634,7 +714,7 @@ dimension = 3072 fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--yes") + output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(err, output) loaded := fixture.load(t) diff --git a/docs/changelog.md b/docs/changelog.md index e81f029f4..51ee3fbcd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -78,9 +78,11 @@ All notable changes to msgvault, grouped by release. configures contextual text search (`voyage-context-4`), semantic people search, and the visual lane once its probe manifest exists; a Mistral key configures document extraction and, with a text lane, document vectors; an - OpenAI key onboards the people sweep on `gpt-5.6-luna` (and the - OpenAI-compatible text lane when no Voyage key is present); with no hosted - key a reachable local Ollama server is used. Setup asks once per hosted + OpenAI key configures fallback text search when no Voyage key is present + and, with `--allow-sensitive`, the people sweep on `gpt-5.6-luna`; with no hosted + key a reachable local Ollama server is used. The sweep requires a separate + `--allow-sensitive` opt-in for sensitive archive excerpts and personal + inferences. Setup asks once per hosted provider, never turns a hosted lane on from a key alone, leaves configured lanes untouched, and prints the next commands. `msgvault setup status` reports every lane with provider, model, consent state, schedule, and the diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 54dadf6f0..30a2cca45 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1915,7 +1915,13 @@ the recommended sections to `config.toml`, onboards the people-sweep provider through the same check and consent gates as `person provider`, and prints the lane report with the next commands. Lanes that are already configured are left alone. See -[Recommended Configuration](/usage/recommended-configuration/). +[Recommended Configuration](/docs/usage/recommended-configuration/). + +The people sweep stays pending unless `--allow-sensitive` is supplied. +This permits sending sensitive archive excerpts to its inference provider +and inferring sensitive personal attributes. `--yes` alone does not grant +this permission. Vector lanes also stay pending when the binary lacks the +backend required by the configured database; setup prints rebuild guidance. ```bash msgvault setup providers --dry-run @@ -1926,6 +1932,7 @@ msgvault setup providers --yes --document-retention zdr --document-training opte | Flag | Default | Description | |---|---|---| | `--yes` | `false` | Accept every provider disclosure without prompting (required when stdin is not a terminal) | +| `--allow-sensitive` | `false` | Allow the people sweep to send sensitive archive excerpts and infer sensitive personal attributes | | `--dry-run` | `false` | Print the plan, the disclosures, and the current lane report without writing | | `--document-retention` | `standard` | Mistral retention posture to record: `standard` or `zdr` | | `--document-training` | `default-opt-out` | Mistral training posture to record: `default-opt-out` or `opted-out` | diff --git a/docs/configuration.md b/docs/configuration.md index addb2e24f..58328fad9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -344,7 +344,7 @@ client_secrets = 'C:\Users\you\Downloads\client_secret.json' `[attachments.documents]`, and `[people.sweep]` sections from the API keys in your environment, and `msgvault setup status` reports every lane with its provider, model, consent state, and next step. The values it chooses are -listed in [Recommended Configuration](/usage/recommended-configuration/). +listed in [Recommended Configuration](/docs/usage/recommended-configuration/). ### `[data]` @@ -1055,7 +1055,7 @@ Optional background scheduling for the embed worker inside `msgvault serve`. Emp | `cron` | — | 5-field cron expression. Empty string disables the standalone cron. | | `run_after_sync` | `false` | When `true`, an embed pass runs after every successful scheduled sync. | -`msgvault setup providers` sets `run_after_sync = true` and `cron = "*/15 * * * *"` when it enables a text lane. See [Recommended Configuration](/usage/recommended-configuration/). +`msgvault setup providers` sets `run_after_sync = true` and `cron = "*/15 * * * *"` when it enables a text lane. See [Recommended Configuration](/docs/usage/recommended-configuration/). #### `[vector.people]` diff --git a/docs/setup.md b/docs/setup.md index 96d7e3b8c..177d6368f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -322,7 +322,10 @@ msgvault setup providers # one consent per provider, then config.toml is msgvault setup status # what is on, what is off, and why ``` -See [Recommended Configuration](/usage/recommended-configuration/) for the +The people sweep additionally requires `msgvault setup providers --allow-sensitive` +to permit sensitive archive excerpts and sensitive personal inferences. + +See [Recommended Configuration](/docs/usage/recommended-configuration/) for the values it writes and the probe steps the hosted lanes still need. ## Optional: Sync Google Calendar diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index 18a4339b9..26f73a60f 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -19,6 +19,7 @@ export OPENAI_API_KEY="..." # people sweep (and text search when no Voyage msgvault setup providers --dry-run # show the plan and each provider disclosure msgvault setup providers # answer once per provider, write config.toml +msgvault setup providers --allow-sensitive # opt into sensitive evidence for the people sweep msgvault setup status # what is on, what is off, and why ``` @@ -31,6 +32,12 @@ adding a key upgrades only the lanes that are still unset; a configured lane keeps its model, because switching the embedding policy invalidates the index and is your call. +The people sweep stays pending without `--allow-sensitive`, even with `--yes`. +The flag permits sending sensitive archive excerpts to the inference provider +and inferring sensitive personal attributes. The plan describes this policy +in both human and JSON output. Vector lanes stay pending if the binary lacks +the backend needed for your database; rebuild as directed before re-running. + Every value below is settable per lane exactly as before. This page only describes what happens when nothing is set. @@ -40,12 +47,13 @@ describes what happens when nothing is set. |---|---|---|---| | `VOYAGE_API_KEY` | text search, semantic people search, visual attachments (after the probe) | `voyage-context-4` (1024), `voyage-multimodal-3.5` (1024) | Chats embed as conversation windows and meetings as turn-aware chunks; email rides the same generation. | | `MISTRAL_API_KEY` | document extraction and lexical search; document vectors when a text lane is on | `mistral-ocr-4-0`, EU region | Uploads are manual-only and need the probe manifest plus `documents consent-mistral --yes`. | -| `OPENAI_API_KEY` | people sweep; text search only when no Voyage key | `gpt-5.6-luna` at `medium` reasoning; `text-embedding-3-small` (1536) | The OpenAI text path gives per-message vectors: no conversation-window context and no visual lane, both are Voyage-only endpoints. | -| none | local Ollama at `[chat].server` when reachable | `nomic-embed-text` (768); the `[chat].model` for the sweep | Text stays on your machine. Setup skips a lane the server cannot serve and says why. | +| `OPENAI_API_KEY` | people sweep with `--allow-sensitive`; text search only when no Voyage key | `gpt-5.6-luna` at `medium` reasoning; `text-embedding-3-small` (1536) | The OpenAI text path gives per-message vectors: no conversation-window context and no visual lane, both are Voyage-only endpoints. | +| none | local Ollama at `[chat].server` when reachable | `nomic-embed-text` (768); the `[chat].model` for the sweep with `--allow-sensitive` | Text stays on your machine. Setup skips a lane the server cannot serve and says why. | ## The file setup writes -With a Voyage key, a Mistral key, and an OpenAI key present, setup writes +With a Voyage key, a Mistral key, and an OpenAI key present and +`--allow-sensitive` supplied, setup writes the sections below into an otherwise empty `config.toml`. Comments and sections you already have are preserved. @@ -116,7 +124,7 @@ text leaves the machine either way; setup states that before it asks. `run_after_sync` covers Gmail, IMAP, Teams, and Discord syncs. The cron covers Slack, Beeper, calendar, and meeting sources, which do not trigger a -post-sync embed. See [Vector Search](/usage/vector-search/). +post-sync embed. See [Vector Search](/docs/usage/vector-search/). ### `[vector.people]` @@ -151,7 +159,7 @@ legal postures (`standard` retention, `default-opt-out` training) unless you pass `--document-retention zdr` or `--document-training opted-out`; use the values your account actually has. Uploads stay manual: build the fixture matrix, probe, consent, then build. See -[Document Attachment Indexing](/usage/document-indexing/). +[Document Attachment Indexing](/docs/usage/document-indexing/). ```bash msgvault documents probe-mistral --fixtures > ~/.msgvault/mistral-capabilities.json @@ -168,8 +176,10 @@ track (`msgvault person track `). Deterministic contact state through `[activity]` and needs no model. Setup onboards the `openai` profile through `person provider add` (a synthetic check request is sent), records consent, and selects it; the daily schedule is the `[people.sweep]` default. -`allow_sensitive = true` is required for real sweeps because every evidence -packet is marked sensitive. The Codex app-server adapter is release-gated and +`allow_sensitive = true` is required for real sweeps because every archive +evidence packet is marked sensitive. Setup sets it only when you pass +`--allow-sensitive`; without that flag it leaves the sweep unconfigured. +The Codex app-server adapter is release-gated and cannot be the default. With no OpenAI key, setup offers a loopback Ollama profile on `[chat].model`. @@ -177,14 +187,14 @@ profile on `[chat].model`. On by default (`17 * * * *`, UTC). It projects archived messages into dated per-person contact state. Nothing to configure; see -[Configuration](/configuration/#activity). +[Configuration](/docs/configuration/#activity). ### Media policy Chat sources cap collection by conversation size: media from rooms above 20 participants is skipped with a typed `participant_threshold` marker, direct and small-group media is kept. Set `media_max_participants = 0` on a source -to lift the cap. See the `[beeper]`, `[slack]`, `[discord]`, and `[teams]` sections of [Configuration](/configuration/#beeper). +to lift the cap. See the `[beeper]`, `[slack]`, `[discord]`, and `[teams]` sections of [Configuration](/docs/configuration/#beeper). ## What the MCP server answers with these defaults @@ -208,4 +218,6 @@ Document attachments (...) on mistral mistral-ocr-4-0 m `pending` means the lane is configured or the key is present but an operator step remains; the `next` line under the table names it. `unknown` consent means the archive could not be read (for example, the database does -not exist yet). Use `--json` for scripting. +not exist yet). Hosted visual, document, and people-sweep lanes also show +`pending` when a required credential environment variable is missing. +Use `--json` for scripting. From e03ba4fb669f50871635e2d9e06c0dfca15a2739 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 4 Sep 2026 21:08:43 -0500 Subject: [PATCH 04/10] fix(setup): preserve provider policies and separate query consent Document search sends both archive text and query text to an embedding provider. Report their separate consent records so users can see which permission is still missing and how to grant it. Enabling a lane must not replace saved retention or training assertions with command defaults. Preserve each assertion unless its own flag is explicitly supplied. Recognize providers by exact URL hosts, not substrings. Leave dependent lanes on custom hosted endpoints for explicit configuration instead of extending the data sent under another provider's disclosure. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/documents_vector_test.go | 33 ++++++++ cmd/msgvault/cmd/setup_lanes.go | 95 ++++++++++++++++------- cmd/msgvault/cmd/setup_providers.go | 61 ++++++++++----- cmd/msgvault/cmd/setup_providers_test.go | 76 ++++++++++++++++++ docs/cli-reference.md | 7 ++ docs/usage/recommended-configuration.md | 17 ++++ 6 files changed, 244 insertions(+), 45 deletions(-) diff --git a/cmd/msgvault/cmd/documents_vector_test.go b/cmd/msgvault/cmd/documents_vector_test.go index 1293f4d97..373eb2c58 100644 --- a/cmd/msgvault/cmd/documents_vector_test.go +++ b/cmd/msgvault/cmd/documents_vector_test.go @@ -119,6 +119,39 @@ func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { assert.Zero(runtimeCalls) } +func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture, _ := documentVectorCommandFixture(t) + cfg.Attachments.Documents.Enabled = true + deps := documentsCommandDeps{ + openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + } + lane := documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + assert.Equal(map[string]string{"document_embedding": consentMissing, "query_embedding": consentMissing}, lane.ConsentPurposes) + assert.ElementsMatch([]string{"msgvault documents vectors consent --yes", "msgvault documents vectors consent --purpose queries --yes"}, lane.Next) + for _, purpose := range []string{"documents", "queries"} { + command := newDocumentsCmd(deps) + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"vectors", "consent", "--purpose", purpose, "--yes"}) + require.NoError(command.ExecuteContext(t.Context()), output.String()) + lane = documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + assert.Equal(consentActive, lane.ConsentPurposes["document_embedding"]) + if purpose == "documents" { + assert.Equal(consentMissing, lane.ConsentPurposes["query_embedding"]) + assert.Equal([]string{"msgvault documents vectors consent --purpose queries --yes"}, lane.Next) + } else { + assert.Equal(consentActive, lane.ConsentPurposes["query_embedding"]) + assert.Equal(consentActive, lane.Consent) + assert.Empty(lane.Next) + } + } + cfg.Vector.Embeddings.Endpoint = "https://changed.example.test/v1" + lane = documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + assert.Equal(map[string]string{"document_embedding": consentMissing, "query_embedding": consentMissing}, lane.ConsentPurposes) +} + func TestDocumentVectorStatusWorksWhenEmbeddingsAreDisabled(t *testing.T) { markDaemonCLISubprocessForTest(t) previous := cfg diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index 27fec566e..3473d04ed 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -5,8 +5,12 @@ import ( "encoding/json" "fmt" "io" + "maps" + "net/netip" + "net/url" "os" "path/filepath" + "slices" "strings" "text/tabwriter" @@ -15,6 +19,7 @@ import ( "go.kenn.io/msgvault/internal/peoplesweep" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" + vectordocument "go.kenn.io/msgvault/internal/vector/document" ) // Lane and consent states shared by `setup providers` and `setup status`. @@ -65,15 +70,16 @@ const ( // laneStatus is one row of the provider report. It answers, for one lane: // which provider and model, whether it is on, why not, and what to run next. type laneStatus struct { - Lane string `json:"lane"` - Label string `json:"label"` - State string `json:"state"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - Schedule string `json:"schedule,omitempty"` - Consent string `json:"consent,omitempty"` - Reason string `json:"reason,omitempty"` - Next []string `json:"next,omitempty"` + Lane string `json:"lane"` + Label string `json:"label"` + State string `json:"state"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + Schedule string `json:"schedule,omitempty"` + Consent string `json:"consent,omitempty"` + ConsentPurposes map[string]string `json:"consent_purposes,omitempty"` + Reason string `json:"reason,omitempty"` + Next []string `json:"next,omitempty"` } // laneReport is the complete report printed by both setup subcommands. @@ -88,10 +94,12 @@ type laneReport struct { // incompatibility, PostgreSQL unreachable) and every consent is reported as // unknown rather than missing. type setupConsentState struct { - Documents bool - Visual bool - PersonInference bool - PersonSemantic bool + Documents bool + Visual bool + PersonInference bool + PersonSemantic bool + DocumentEmbedding bool + QueryEmbedding bool } // setupEnvironment is what the report needs beyond the loaded config: the @@ -171,6 +179,18 @@ func readSetupConsentState(ctx context.Context, cfg *config.Config) *setupConsen // hide the others. func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.Store) *setupConsentState { state := &setupConsentState{} + if cfg.Vector.Enabled && cfg.Attachments.Documents.Index.Embeddings.Enabled { + if target, err := st.GetDocumentVectorTargetProfileID(ctx); err == nil { + if fingerprint, err := vectordocument.EgressFingerprint(target, cfg.Vector); err == nil { + consent, err := st.GetDocumentVectorConsent(ctx, fingerprint) + state.DocumentEmbedding = err == nil && consent != nil && consent.Purpose == "document_embedding" + } + if fingerprint, err := vectordocument.QueryEgressFingerprint(target, cfg.Vector); err == nil { + consent, err := st.GetDocumentVectorConsent(ctx, fingerprint) + state.QueryEmbedding = err == nil && consent != nil && consent.Purpose == "query_embedding" + } + } + } if consented, err := st.HasActiveDocumentProviderConsent(ctx); err == nil { state.Documents = consented } @@ -198,19 +218,27 @@ func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.St // embeddingProviderName names the embedding destination for the report. func embeddingProviderName(endpoint string) string { - lower := strings.ToLower(endpoint) - switch { - case strings.Contains(lower, "voyageai.com"): - return "voyage" - case strings.Contains(lower, "openai.com"): - return "openai" - case strings.Contains(lower, "localhost") || strings.Contains(lower, "127.0.0.1") || strings.Contains(lower, "::1"): - return "local" - case endpoint == "": + if endpoint == "" { return "" - default: + } + parsed, err := url.Parse(endpoint) + if err != nil || parsed.User != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { return "custom" } + host := strings.ToLower(parsed.Hostname()) + address, _ := netip.ParseAddr(host) + if host == "localhost" || address.Unmap().IsLoopback() { + return "local" + } + if parsed.Scheme == "https" && (parsed.Port() == "" || parsed.Port() == "443") { + switch host { + case "api.voyageai.com": + return "voyage" + case "api.openai.com": + return "openai" + } + } + return "custom" } func embedScheduleSummary(schedule vector.EmbedScheduleConfig) string { @@ -236,7 +264,7 @@ func buildLaneReport(cfg *config.Config, env setupEnvironment) laneReport { personSearchLane(cfg, env), visualSearchLane(cfg, env), documentsLane(cfg, env), - documentVectorsLane(cfg), + documentVectorsLane(cfg, env), peopleInferenceLane(cfg, env), activityLane(cfg), mediaPolicyLane(cfg), @@ -381,7 +409,7 @@ func documentsLane(cfg *config.Config, env setupEnvironment) laneStatus { return lane } -func documentVectorsLane(cfg *config.Config) laneStatus { +func documentVectorsLane(cfg *config.Config, env setupEnvironment) laneStatus { lane := laneStatus{Lane: laneDocumentVectors, Label: "Document semantic search"} documents := cfg.Attachments.Documents switch { @@ -390,8 +418,18 @@ func documentVectorsLane(cfg *config.Config) laneStatus { lane.Provider = embeddingProviderName(cfg.Vector.Embeddings.Endpoint) lane.Model = cfg.Vector.Embeddings.Model lane.Schedule = embedScheduleSummary(cfg.Vector.Embed.Schedule) - lane.Reason = "document chunks are embedded with the text-search profile after a separate consent" - lane.Next = []string{"msgvault documents vectors consent --yes"} + lane.Reason = "document chunks and search query text are sent to the text-search provider under separate consents" + lane.ConsentPurposes = map[string]string{ + "document_embedding": env.consentState(func(s setupConsentState) bool { return s.DocumentEmbedding }), + "query_embedding": env.consentState(func(s setupConsentState) bool { return s.QueryEmbedding }), + } + lane.Consent = env.consentState(func(s setupConsentState) bool { return s.DocumentEmbedding && s.QueryEmbedding }) + if lane.ConsentPurposes["document_embedding"] != consentActive { + lane.Next = append(lane.Next, "msgvault documents vectors consent --yes") + } + if lane.ConsentPurposes["query_embedding"] != consentActive { + lane.Next = append(lane.Next, "msgvault documents vectors consent --purpose queries --yes") + } case documents.Enabled && !cfg.Vector.Enabled: lane.State = laneStateOff lane.Reason = "requires the text-search lane" @@ -513,6 +551,9 @@ func writeLaneReport(w io.Writer, report laneReport, jsonOutput bool) error { continue } _, _ = fmt.Fprintf(w, "%s: %s\n", lane.Label, lane.Reason) + for _, purpose := range slices.Sorted(maps.Keys(lane.ConsentPurposes)) { + _, _ = fmt.Fprintf(w, " %s consent: %s\n", purpose, lane.ConsentPurposes[purpose]) + } for _, next := range lane.Next { _, _ = fmt.Fprintf(w, " next: %s\n", next) } diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 581c6f7e4..2ffb1cad6 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -9,7 +9,6 @@ import ( "io" "maps" "net/http" - "net/url" "os" "slices" "sort" @@ -17,6 +16,7 @@ import ( "text/tabwriter" "time" + "github.com/BurntSushi/toml" "github.com/charmbracelet/x/term" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/config" @@ -47,14 +47,16 @@ const ( // setupProvidersOptions are the command flags. type setupProvidersOptions struct { - yes bool - dryRun bool - jsonOutput bool - allowSensitive bool - documentRetention string - documentTraining string - retentionPosture string - trainingPosture string + yes bool + dryRun bool + jsonOutput bool + allowSensitive bool + documentRetention string + documentTraining string + retentionPosture string + trainingPosture string + personRetentionPosture string + personTrainingPosture string } // ollamaProbeResult is what a local Ollama server reports about itself. @@ -238,10 +240,7 @@ func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setup server := strings.TrimRight(strings.TrimSpace(loaded.Chat.Server), "/") detection.ollama = deps.probeOllama(ctx, server) detection.ollamaEndpoint = server + "/v1" - if parsed, err := url.Parse(server); err == nil { - host := parsed.Hostname() - detection.ollamaLoopback = strings.EqualFold(host, "localhost") || host == "127.0.0.1" || host == "::1" - } + detection.ollamaLoopback = embeddingProviderName(server) == "local" } return detection } @@ -517,6 +516,9 @@ func planPersonSearch(loaded *config.Config, plan *setupProvidersPlan, options s case !plan.laneOn(laneTextSearch): lane.Action = planActionSkip lane.Reason = "requires the text-search lane" + case textLaneProvider(plan) == "custom": + lane.Action = planActionSkip + lane.Reason = "custom hosted provider: configure [vector.people] and its postures explicitly, then run `msgvault person provider consent --semantic-embeddings --yes`" default: lane.Action = planActionEnable lane.Gate = textLaneGate(plan) @@ -524,7 +526,7 @@ func planPersonSearch(loaded *config.Config, plan *setupProvidersPlan, options s lane.edits = []config.TableEdit{{ Path: []string{tomlTableVector, "people"}, Values: map[string]any{ - "enabled": true, "retention_posture": options.retentionPosture, "training_posture": options.trainingPosture, + "enabled": true, "retention_posture": options.personRetentionPosture, "training_posture": options.personTrainingPosture, }, }} lane.next = []string{"msgvault person provider consent --semantic-embeddings --yes"} @@ -614,14 +616,17 @@ func planDocumentVectors(loaded *config.Config, plan *setupProvidersPlan) setupL case !plan.laneOn(laneTextSearch): lane.Action = planActionSkip lane.Reason = "requires the text-search lane" + case textLaneProvider(plan) == "custom": + lane.Action = planActionSkip + lane.Reason = "custom hosted provider: configure [attachments.documents.index.embeddings] explicitly and consent separately to document and query text" default: lane.Action = planActionEnable lane.Gate = textLaneGate(plan) - lane.Reason = "extracted document chunks are embedded with the text-search profile after `documents vectors consent`" + lane.Reason = "document chunks and search query text are sent to the text-search provider after separate document and query consents" lane.edits = []config.TableEdit{{ Path: []string{"attachments", "documents", "index", "embeddings"}, Values: map[string]any{"enabled": true}, }} - lane.next = []string{"msgvault documents vectors consent --yes"} + lane.next = []string{"msgvault documents vectors consent --yes", "msgvault documents vectors consent --purpose queries --yes"} } return lane } @@ -724,7 +729,8 @@ func gateDisclosure(gate string, plan *setupProvidersPlan) string { lines = append(lines, " - eligible image and video attachment bytes with bounded message context, after `msgvault multimodal build --yes`") } if plan.laneOn(laneDocumentVectors) && textLaneProvider(plan) == "voyage" { - lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`") + lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`", + " - document search query text, after `msgvault documents vectors consent --purpose queries --yes`") } case gateMistral: lines = append(lines, @@ -740,7 +746,8 @@ func gateDisclosure(gate string, plan *setupProvidersPlan) string { lines = append(lines, " - one curated, non-sensitive attribute document per person") } if plan.laneOn(laneDocumentVectors) && textLaneProvider(plan) == "openai" { - lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`") + lines = append(lines, " - extracted document text, after `msgvault documents vectors consent --yes`", + " - document search query text, after `msgvault documents vectors consent --purpose queries --yes`") } if plan.inference != nil && plan.inference.gate == gateOpenAI { lines = append(lines, " - bounded evidence packets of "+ @@ -861,6 +868,17 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options if err != nil { return err } + // Read saved assertions before config defaults turn an absent document + // posture into "unknown". Only the corresponding explicit flag replaces + // an existing assertion; inference still uses its own command defaults. + var saved config.Config + if _, err := toml.Decode(string(before.Content), &saved); err != nil { + return fmt.Errorf("read saved provider postures: %w", err) + } + options.personRetentionPosture = setupPosture(saved.Vector.People.RetentionPosture, options.retentionPosture, command.Flags().Changed("retention-posture")) + options.personTrainingPosture = setupPosture(saved.Vector.People.TrainingPosture, options.trainingPosture, command.Flags().Changed("training-posture")) + options.documentRetention = setupPosture(saved.Attachments.Documents.RetentionPosture, options.documentRetention, command.Flags().Changed("document-retention")) + options.documentTraining = setupPosture(saved.Attachments.Documents.TrainingPosture, options.documentTraining, command.Flags().Changed("document-training")) now := time.Now() if deps.now != nil { now = deps.now() @@ -921,6 +939,13 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options return writeSetupProvidersResult(command, deps, nil, &plan, options, plan.writes(), declined) } +func setupPosture(saved, proposed string, override bool) string { + if saved != "" && !override { + return saved + } + return proposed +} + // The people-provider commands need a published profile for daemon-owned // checks and consent. Keep their existing gates and restore the entire setup // configuration if any step fails. Retain each publication's identity so a diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 1a4e77f68..814484f8b 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -177,6 +177,82 @@ const setupProvidersMinimalConfig = `# operator comment survives setup data_dir = "{{DIR}}/data" ` +func TestSetupProvidersPreservesPosturesUnlessFlagsOverride(t *testing.T) { + for name, override := range map[string]bool{"preserve": false, "override": true} { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` +[vector.people] +retention_posture = "zero_data_retention" +training_posture = "no_training" +[attachments.documents] +retention_posture = "zdr" +training_posture = "opted-out" +`) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + args := []string{"providers", "--yes"} + peopleRetention, documentTraining := "zero_data_retention", "opted-out" + if override { + args = append(args, "--retention-posture", setupPostureDeclared, "--document-training", documentindex.TrainingDefaultOptOut) + peopleRetention, documentTraining = setupPostureDeclared, documentindex.TrainingDefaultOptOut + } + output, err := fixture.run(t, args...) + require.NoError(t, err, output) + loaded := fixture.load(t) + assert.Equal(peopleRetention, loaded.Vector.People.RetentionPosture) + assert.Equal("no_training", loaded.Vector.People.TrainingPosture) + assert.Equal("zdr", loaded.Attachments.Documents.RetentionPosture) + assert.Equal(documentTraining, loaded.Attachments.Documents.TrainingPosture) + assert.Contains(output, "retention=zdr, training="+documentTraining) + }) + } +} + +func TestSetupProvidersCustomHostedEndpointNeedsExplicitConfiguration(t *testing.T) { + for _, endpoint := range []string{"https://api.openai.com.example.test/v1", "https://localhost.example.test/v1", "https://embeddings.example.test/v1"} { + t.Run(endpoint, func(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+fmt.Sprintf(` +[vector] +enabled = true +[vector.embeddings] +endpoint = %q +model = "embedding-test" +dimension = 768 +[attachments.documents] +enabled = true +retention_posture = "zdr" +training_posture = "opted-out" +`, endpoint)) + before := fixture.readConfig(t) + output, err := fixture.run(t, "providers", "--yes") + require.NoError(t, err, output) + assert.Equal(before, fixture.readConfig(t)) + assert.False(fixture.load(t).Vector.People.Enabled) + assert.False(fixture.load(t).Attachments.Documents.Index.Embeddings.Enabled) + assert.Contains(output, "custom hosted provider") + }) + } +} + +func TestEmbeddingProviderNameUsesURLHost(t *testing.T) { + for endpoint, want := range map[string]string{ + setupOpenAIEndpoint: "openai", setupVoyageEndpoint: "voyage", + "https://API.OPENAI.COM:443/v1": "openai", + "http://api.openai.com/v1": "custom", + "https://api.openai.com:8443/v1": "custom", + "http://localhost:11434/v1": "local", "http://127.0.0.2:11434/v1": "local", + "http://[::1]:11434/v1": "local", + "https://localhost.example.test/v1": "custom", + "https://example.test/voyageai.com?host=127.0.0.1": "custom", + "https://api.openai.com@example.test/v1": "custom", + "not a URL": "custom", "": "", + } { + t.Run(endpoint, func(t *testing.T) { assert.Equal(t, want, embeddingProviderName(endpoint)) }) + } +} + func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 30a2cca45..73b05c044 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1923,6 +1923,13 @@ and inferring sensitive personal attributes. `--yes` alone does not grant this permission. Vector lanes also stay pending when the binary lacks the backend required by the configured database; setup prints rebuild guidance. +Saved retention and training postures on disabled lanes are preserved unless +their corresponding posture flags are explicitly supplied. Custom hosted +text endpoints require explicit configuration of dependent lanes. Document +semantic search requires both `documents vectors consent --yes` and +`documents vectors consent --purpose queries --yes`; the latter authorizes +query-text uploads. The status report tracks the two purposes separately. + ```bash msgvault setup providers --dry-run msgvault setup providers diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index 26f73a60f..ae3ce8c43 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -32,6 +32,17 @@ adding a key upgrades only the lanes that are still unset; a configured lane keeps its model, because switching the embedding policy invalidates the index and is your call. +When enabling a disabled lane, setup preserves saved retention and training +postures. Defaults fill only unset values. Pass `--retention-posture` or +`--training-posture` to replace the corresponding people-search posture, +or `--document-retention` or `--document-training` for document extraction. +Already enabled lanes remain unchanged. + +For existing text endpoints, setup recognizes the exact OpenAI and Voyage +API hosts over HTTPS and loopback servers. Other hosted endpoints are custom: +configure their people-search and document-vector lanes explicitly, then +review the separate consent commands for the new data they will receive. + The people sweep stays pending without `--allow-sensitive`, even with `--yes`. The flag permits sending sensitive archive excerpts to the inference provider and inferring sensitive personal attributes. The plan describes this policy @@ -166,8 +177,14 @@ msgvault documents probe-mistral --fixtures > ~/.msgvault/ msgvault documents consent-mistral --capabilities ~/.msgvault/mistral-capabilities.json --yes msgvault documents build --capabilities ~/.msgvault/mistral-capabilities.json --yes msgvault documents vectors consent --yes # when document vectors are enabled +msgvault documents vectors consent --purpose queries --yes ``` +Document-vector consent covers document text. Query consent separately +permits sending semantic and hybrid document search query text to the +embedding provider. `setup status` reports `document_embedding` and +`query_embedding` consent separately and lists each missing consent command. + ### `[people.sweep]` The sweep keeps curated attributes current from the archive for people you From f06f780da2706bbad4e61058752c70f5a5ee655e Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 4 Sep 2026 21:30:29 -0500 Subject: [PATCH 05/10] fix(setup): honor readiness and schedule opt-outs Missing embedding credentials make text search unavailable to its people and document consumers too. Report those lanes as pending instead of on. Preserve explicit empty cron and false after-sync settings when applying text and visual defaults so setup does not undo scheduling opt-outs. A running daemon may lack the caller's provider key during onboarding. Forward only that key for a check pinned to the saved profile fingerprint, and preserve the caller context on the generated command. Keep ordinary checks on daemon-owned credentials and reject keys for other profiles. Generated with Codex Co-authored-by: Codex --- .../cmd/person_provider_daemon_test.go | 95 ++++++++++++++++++- cmd/msgvault/cmd/person_provider_setup.go | 29 +++++- cmd/msgvault/cmd/setup_lanes.go | 12 ++- cmd/msgvault/cmd/setup_providers.go | 39 ++++---- cmd/msgvault/cmd/setup_providers_test.go | 33 +++++++ docs/usage/recommended-configuration.md | 6 ++ internal/api/cli_handlers.go | 36 +++++-- 7 files changed, 218 insertions(+), 32 deletions(-) diff --git a/cmd/msgvault/cmd/person_provider_daemon_test.go b/cmd/msgvault/cmd/person_provider_daemon_test.go index 26f77ff15..5a4e3bf1e 100644 --- a/cmd/msgvault/cmd/person_provider_daemon_test.go +++ b/cmd/msgvault/cmd/person_provider_daemon_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync/atomic" "testing" @@ -53,7 +54,12 @@ func (s *inProcessPersonProviderDaemonStore) RunCLICommand( config, consent, registry, - peoplesweep.NewCredentialResolver(s.credentials, os.LookupEnv), + peoplesweep.NewCredentialResolver(s.credentials, func(name string) (string, bool) { + if value, ok := req.Env[name]; ok { + return value, true + } + return os.LookupEnv(name) + }), ) } @@ -77,6 +83,93 @@ func (s *inProcessPersonProviderDaemonStore) RunCLICommand( return nil } +func TestSavedPersonProviderCheckForwardsExactCredentialThroughDaemon(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + const keyName = "SETUP_ONLY_PROVIDER_KEY" + const secret = "synthetic-onboarding-key" + t.Setenv(keyName, "") // The daemon process does not have the caller's key. + var received atomic.Int64 + provider := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+secret { + http.Error(w, "missing credential", http.StatusUnauthorized) + return + } + received.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"model":"test-model","choices":[{"message":{"content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`) + })) + t.Cleanup(provider.Close) + peopleConfig := personProviderTestConfig() + onboarded := configuredPersonProvider(peopleConfig) + onboarded.Endpoint, onboarded.CredentialEnv = provider.URL+"/v1", keyName + peopleConfig.Providers["onboarded"] = onboarded + daemonConfig := config.NewDefaultConfig() + daemonConfig.HomeDir = t.TempDir() + daemonConfig.People.Sweep = personProviderTestConfig() + // Onboarding publishes a new profile after the daemon has started. + saved := *daemonConfig + saved.People.Sweep = peopleConfig + require.NoError(saved.Save()) + st := &inProcessPersonProviderDaemonStore{ + storeAPIAdapter: &storeAPIAdapter{store: testutil.NewSQLiteTestStore(t)}, + config: peopleConfig, httpClient: provider.Client(), + } + daemon := api.NewServerWithOptions(api.ServerOptions{ + Config: daemonConfig, Store: st, Logger: slog.New(slog.DiscardHandler), + OperationGate: api.NewSerialOperationGate(), + }) + server := httptest.NewServer(daemon.Router()) + t.Cleanup(server.Close) + frontend := *daemonConfig + frontend.People.Sweep = peopleConfig + frontend.Remote = config.RemoteConfig{URL: server.URL, AllowInsecure: true} + withStoreResolverConfig(t, &frontend) + deps := defaultPersonProviderCommandDeps() + callerHasKey := false + deps.setup.lookupEnv = func(name string) (string, bool) { + assert.Equal(keyName, name) + return secret, callerHasKey + } + var output bytes.Buffer + command := &cobra.Command{Use: "setup"} + command.SetContext(t.Context()) + command.SetOut(&output) + command.SetErr(&output) + require.Error(executeSavedPersonProviderCheck(command, deps, "onboarded", "", &output)) + assert.Zero(received.Load()) + output.Reset() + callerHasKey = true + require.NoError(executeSavedPersonProviderCheck(command, deps, "onboarded", "", &output), output.String()) + assert.Equal(int64(1), received.Load()) + assert.NotContains(output.String(), secret) + + profileConfig := peopleConfig + profileConfig.Enabled = true + profileConfig.Provider = peoplesweep.ProviderSelection{Name: "onboarded"} + profile, err := profileConfig.Profile() + require.NoError(err) + for _, test := range []struct{ name, fingerprint, key string }{ + {name: "other provider key", fingerprint: profile.Fingerprint, key: "TEST_PROVIDER_KEY"}, + {name: "changed profile", fingerprint: strings.Repeat("a", 64), key: keyName}, + {name: "ordinary check", key: keyName}, + } { + t.Run(test.name, func(t *testing.T) { + args := []string{"person", "provider", "check", "onboarded"} + if test.fingerprint != "" { + args = append(args, "--if-fingerprint", test.fingerprint) + } + body := mustJSON(t, api.CLIRunRequest{Args: args, Env: map[string]string{test.key: secret}}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/cli/run", bytes.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + daemon.Router().ServeHTTP(response, request) + assert.Equal(http.StatusBadRequest, response.Code, response.Body.String()) + assert.Equal(int64(1), received.Load()) + }) + } +} + func TestPersonProviderRealDaemonSyntheticCheckAndRevoke(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/cmd/msgvault/cmd/person_provider_setup.go b/cmd/msgvault/cmd/person_provider_setup.go index b6e72a578..69a4bd597 100644 --- a/cmd/msgvault/cmd/person_provider_setup.go +++ b/cmd/msgvault/cmd/person_provider_setup.go @@ -1026,7 +1026,26 @@ func executeSavedPersonProviderCheck( } return writePersonProviderCheckOutput(out, output, false) } - return proxySavedPersonProviderOperation(command, deps, "check", name, ifFingerprint, out) + selected, err := selectPersonProviderConfig(deps.config(), name) + if err != nil { + return err + } + selected.Enabled = true + profile, err := selected.Profile() + if err != nil { + return err + } + if ifFingerprint != "" && ifFingerprint != profile.Fingerprint { + return errors.New("people provider profile changed before checking") + } + var env map[string]string + if profile.Credential == peoplesweep.CredentialEnv && profile.Auth != peoplesweep.AuthNone && deps.setup.lookupEnv != nil { + if value, ok := deps.setup.lookupEnv(profile.CredentialRef); ok && strings.TrimSpace(value) != "" { + env = map[string]string{profile.CredentialRef: value} + } + } + return proxySavedPersonProviderOperationWithFlag(command, deps, "check", name, + personProviderIfFingerprintFlag, profile.Fingerprint, out, env) } func verifyPersonProviderFingerprint( @@ -1070,7 +1089,7 @@ func proxySavedPersonProviderRevokeFingerprint( fingerprint string, ) error { return proxySavedPersonProviderOperationWithFlag( - command, deps, "revoke", name, "fingerprint", fingerprint, io.Discard, + command, deps, "revoke", name, "fingerprint", fingerprint, io.Discard, nil, ) } @@ -1083,7 +1102,7 @@ func proxySavedPersonProviderOperation( out io.Writer, ) error { return proxySavedPersonProviderOperationWithFlag( - command, deps, operation, name, personProviderIfFingerprintFlag, fingerprint, out, + command, deps, operation, name, personProviderIfFingerprintFlag, fingerprint, out, nil, ) } @@ -1095,6 +1114,7 @@ func proxySavedPersonProviderOperationWithFlag( flag string, fingerprint string, out io.Writer, + env map[string]string, ) error { if err := peoplesweep.ValidateProviderProfileName(name); err != nil { return err @@ -1123,7 +1143,8 @@ func proxySavedPersonProviderOperationWithFlag( root.AddCommand(person) leaf.SetOut(out) leaf.SetErr(command.ErrOrStderr()) - return deps.proxy(leaf, []string{name}, nil) + leaf.SetContext(command.Context()) + return deps.proxy(leaf, []string{name}, env) } func rollbackNewPersonProviderCredential( diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index 3473d04ed..4ca7d5f07 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -286,9 +286,7 @@ func textSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { } else { lane.Reason = "per-message vectors; no conversation-window context (Voyage contextual only)" } - if embeddings.APIKeyEnv != "" && !env.hasEnv(embeddings.APIKeyEnv) { - lane.Reason += "; environment variable " + embeddings.APIKeyEnv + " is not set" - } + env.reportMissingCredential(&lane, embeddings.APIKeyEnv) return lane } lane.State = laneStateOff @@ -315,6 +313,10 @@ func personSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Model = cfg.Vector.Embeddings.Model lane.Consent = env.consentState(func(s setupConsentState) bool { return s.PersonSemantic }) lane.Reason = "one curated document per person rides the text-search generation" + if text := textSearchLane(cfg, env); text.State != laneStateOn { + lane.State = laneStatePending + lane.Reason += "; text search is not ready: " + text.Reason + } if lane.Consent != consentActive { lane.Next = []string{"msgvault person provider consent --semantic-embeddings --yes"} } @@ -419,6 +421,10 @@ func documentVectorsLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Model = cfg.Vector.Embeddings.Model lane.Schedule = embedScheduleSummary(cfg.Vector.Embed.Schedule) lane.Reason = "document chunks and search query text are sent to the text-search provider under separate consents" + if text := textSearchLane(cfg, env); text.State != laneStateOn { + lane.State = laneStatePending + lane.Reason += "; text search is not ready: " + text.Reason + } lane.ConsentPurposes = map[string]string{ "document_embedding": env.consentState(func(s setupConsentState) bool { return s.DocumentEmbedding }), "query_embedding": env.consentState(func(s setupConsentState) bool { return s.QueryEmbedding }), diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 2ffb1cad6..659dfa002 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -193,6 +193,7 @@ func (r ollamaProbeResult) hasModel(name string) bool { // the local Ollama server offers, which probe manifests are already written, // and which vector backend the archive selects. type setupDetection struct { + configKeys toml.MetaData voyageKey bool mistralKey bool mistralKeyEnv string @@ -423,15 +424,18 @@ func textLaneGateForProvider(provider string) string { } } -func embedScheduleEdit(loaded *config.Config) []config.TableEdit { - schedule := loaded.Vector.Embed.Schedule - if schedule.Cron != "" || schedule.RunAfterSync { +func setupScheduleEdits(keys toml.MetaData, lane string) []config.TableEdit { + path := []string{tomlTableVector, lane, "schedule"} + values := map[string]any{} + for name, value := range map[string]any{"run_after_sync": true, "cron": setupEmbedCron} { + if !keys.IsDefined(tomlTableVector, lane, "schedule", name) { + values[name] = value + } + } + if len(values) == 0 { return nil } - return []config.TableEdit{{ - Path: []string{tomlTableVector, "embed", "schedule"}, - Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}, - }} + return []config.TableEdit{{Path: path, Values: values}} } func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePlan { @@ -464,7 +468,7 @@ func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePl "api_format": "voyage-contextual", "endpoint": setupVoyageEndpoint, "api_key_env": setupVoyageKeyEnv, "model": setupVoyageTextModel, "dimension": setupVoyageTextDim, }, - }}, embedScheduleEdit(loaded)...) + }}, setupScheduleEdits(detection.configKeys, "embed")...) case detection.openAIKey: lane.Action, lane.Provider, lane.Model, lane.Gate = planActionEnable, "openai", setupOpenAITextModel, gateOpenAI lane.Reason = "per-message vectors; no conversation-window context and no visual lane, both are Voyage-only" @@ -474,7 +478,7 @@ func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePl "api_format": "openai", "endpoint": setupOpenAIEndpoint, "api_key_env": setupOpenAIKeyEnv, "model": setupOpenAITextModel, "dimension": setupOpenAITextDim, }, - }}, embedScheduleEdit(loaded)...) + }}, setupScheduleEdits(detection.configKeys, "embed")...) case detection.ollama.Reachable && !detection.ollamaLoopback: // A reachable server that is not on this machine would receive // message text without a credential or a disclosure; only the operator @@ -493,7 +497,7 @@ func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePl "document_prefix": setupOllamaDocPrefix, "query_prefix": setupOllamaQueryPrefix, "max_input_chars": setupOllamaMaxInput, }, - }}, embedScheduleEdit(loaded)...) + }}, setupScheduleEdits(detection.configKeys, "embed")...) case detection.ollama.Reachable: lane.Action = planActionSkip lane.Reason = "Ollama is reachable but has no " + setupOllamaTextModel + "; run `ollama pull " + setupOllamaTextModel + "` or set " + setupVoyageKeyEnv @@ -549,17 +553,14 @@ func planVisualSearch(loaded *config.Config, detection setupDetection) setupLane case detection.voyageManifest != "": lane.Action, lane.Gate = planActionEnable, gateVoyage lane.Reason = "probe manifest found at " + detection.voyageManifest - lane.edits = []config.TableEdit{ + lane.edits = append([]config.TableEdit{ {Path: []string{tomlTableVector, "multimodal"}, Values: map[string]any{"enabled": true, "capabilities_file": detection.voyageManifest}}, - {Path: []string{tomlTableVector, "multimodal", "schedule"}, Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}}, - } + }, setupScheduleEdits(detection.configKeys, "multimodal")...) lane.next = []string{"msgvault multimodal build --yes"} default: lane.Action = planActionPending lane.Reason = "the provider probe needs private synthetic WebP and MP4 seeds; the lane stays off until the manifest exists" - lane.edits = []config.TableEdit{ - {Path: []string{tomlTableVector, "multimodal", "schedule"}, Values: map[string]any{"run_after_sync": true, "cron": setupEmbedCron}}, - } + lane.edits = setupScheduleEdits(detection.configKeys, "multimodal") lane.next = []string{visualProbeCommand(loaded), "msgvault setup providers"} } return lane @@ -872,8 +873,9 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options // posture into "unknown". Only the corresponding explicit flag replaces // an existing assertion; inference still uses its own command defaults. var saved config.Config - if _, err := toml.Decode(string(before.Content), &saved); err != nil { - return fmt.Errorf("read saved provider postures: %w", err) + keys, err := toml.Decode(string(before.Content), &saved) + if err != nil { + return fmt.Errorf("read saved provider settings: %w", err) } options.personRetentionPosture = setupPosture(saved.Vector.People.RetentionPosture, options.retentionPosture, command.Flags().Changed("retention-posture")) options.personTrainingPosture = setupPosture(saved.Vector.People.TrainingPosture, options.trainingPosture, command.Flags().Changed("training-posture")) @@ -884,6 +886,7 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options now = deps.now() } detection := detectSetupProviders(ctx, loaded, deps) + detection.configKeys = keys plan := planSetupProviders(loaded, detection, options, now) out := command.OutOrStdout() if !options.jsonOutput { diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 814484f8b..386218673 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -312,6 +312,7 @@ func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { _, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(err) for lane, key := range map[string]string{ + laneTextSearch: setupVoyageKeyEnv, lanePersonSearch: setupVoyageKeyEnv, laneDocumentVectors: setupVoyageKeyEnv, laneVisualSearch: setupVoyageKeyEnv, laneDocuments: "MISTRAL_API_KEY", lanePeopleInference: setupOpenAIKeyEnv, } { fixture.env[key] = " " @@ -331,6 +332,38 @@ func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { } } +func TestSetupProvidersPreservesExplicitSchedules(t *testing.T) { + for _, manifest := range []bool{false, true} { + for _, schedule := range []struct { + name, toml, cron string + runAfterSync bool + }{ + {name: "manual", toml: "cron = \"\"\nrun_after_sync = false"}, + {name: "sync only", toml: "cron = \"\"\nrun_after_sync = true", runAfterSync: true}, + {name: "custom cron", toml: "cron = \"15 4 * * *\"\nrun_after_sync = false", cron: "15 4 * * *"}, + {name: "unset", cron: setupEmbedCron, runAfterSync: true}, + {name: "cron disabled", toml: "cron = \"\"", runAfterSync: true}, + {name: "sync disabled", toml: "run_after_sync = false", cron: setupEmbedCron}, + } { + t.Run(fmt.Sprintf("%s/manifest=%t", schedule.name, manifest), func(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+ + "\n[vector.embed.schedule]\n"+schedule.toml+ + "\n[vector.multimodal.schedule]\n"+schedule.toml+"\n") + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = manifest + output, err := fixture.run(t, "providers", "--yes") + require.NoError(t, err, output) + loaded := fixture.load(t) + assert.Equal(schedule.cron, loaded.Vector.Embed.Schedule.Cron) + assert.Equal(schedule.runAfterSync, loaded.Vector.Embed.Schedule.RunAfterSync) + assert.Equal(schedule.cron, loaded.Vector.Multimodal.Schedule.Cron) + assert.Equal(schedule.runAfterSync, loaded.Vector.Multimodal.Schedule.RunAfterSync) + }) + } + } +} + func TestSetupProvidersVoyageWritesRecommendedDefaults(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index ae3ce8c43..dbf6c39f7 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -38,6 +38,12 @@ postures. Defaults fill only unset values. Pass `--retention-posture` or or `--document-retention` or `--document-training` for document extraction. Already enabled lanes remain unchanged. +Setup also preserves each explicit `cron` and `run_after_sync` setting for +text and visual embeddings, including `cron = ""` and `run_after_sync = false`. +Only absent keys receive schedule defaults. Missing embedding credentials +leave text search and its dependent people-search and document-vector lanes +pending in the status report. + For existing text endpoints, setup recognizes the exact OpenAI and Voyage API hosts over HTTPS and loopback servers. Other hosted endpoints are custom: configure their people-search and document-vector lanes explicitly, then diff --git a/internal/api/cli_handlers.go b/internal/api/cli_handlers.go index cd09a21b2..4c15e59e0 100644 --- a/internal/api/cli_handlers.go +++ b/internal/api/cli_handlers.go @@ -1344,10 +1344,7 @@ func (s *Server) cliRunEnvAllowedForCommand(args []string, name string) bool { return keyEnv != "" && keyEnv == name } if providerCall { - // Provider checks always resolve credentials from the daemon - // process's own environment and credential store; request-carried - // values are rejected. - return false + return s.cliRunSavedProviderCheckEnvAllowed(args, name) } enrichmentRun := args[1] == "enrichment" && args[2] == "run" if enrichmentRun { @@ -1365,6 +1362,33 @@ func (s *Server) cliRunEnvAllowedForCommand(args []string, name string) bool { return s.cliRunEnvAllowed(name) } +// Local onboarding may supply the selected profile's key for one synthetic +// check. Ordinary checks continue to use daemon-owned credentials. +func (s *Server) cliRunSavedProviderCheckEnvAllowed(args []string, name string) bool { + fingerprint, ok := cliRunFlagValue(args, "if-fingerprint") + if !ok || !cliRunPersonProviderArgsAllowed("check", args[3:]) { + return false + } + var profileName string + for i := 3; i < len(args); i++ { + if args[i] == "--if-fingerprint" { + i++ + } else if !strings.HasPrefix(args[i], "--") { + profileName = args[i] + } + } + sweep, ok := s.currentPeopleSweepConfig() + if !ok || profileName == "" { + return false + } + sweep.Enabled = true + sweep.Provider = peoplesweep.ProviderSelection{Name: profileName} + profile, err := sweep.Profile() + return err == nil && profile.Fingerprint == fingerprint && + profile.Auth != peoplesweep.AuthNone && profile.Credential == peoplesweep.CredentialEnv && + profile.CredentialRef == name +} + func (s *Server) cliRunPersonEnrichmentRunEnvAllowed(args []string, name string) bool { if s.cfg == nil { return false @@ -1911,8 +1935,8 @@ func newCLINDJSONEventWriter[T any](w http.ResponseWriter) func(T) error { } // cliRunEnvAllowed permits the static forwarding allowlist plus configured -// provider variables used by non-check commands. Provider checks always use -// the daemon process's own environment and reject request-carried values. +// provider variables used by non-check commands. Provider checks use the +// separate fingerprint-bound policy in cliRunSavedProviderCheckEnvAllowed. func (s *Server) cliRunEnvAllowed(name string) bool { if clirun.EnvAllowed(name) { return true From 29dffae6ee2a5561101057b5e6224ea809da8abd Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 5 Sep 2026 06:51:28 -0500 Subject: [PATCH 06/10] fix(setup): keep unconsented lanes pending Enabled configuration does not make a consent-gated lane ready. Report missing or unreadable consent as pending, and keep document vectors pending until both document and query consent are active. Local Ollama selection uses an unauthenticated endpoint. Clear the old embedding credential reference so an unused hosted key cannot leave the new local lane pending. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/documents_vector_test.go | 16 +++++-- cmd/msgvault/cmd/setup_lanes.go | 13 +++++- cmd/msgvault/cmd/setup_providers.go | 3 +- cmd/msgvault/cmd/setup_providers_test.go | 57 ++++++++++++++++++++++- docs/usage/recommended-configuration.md | 5 ++ 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/cmd/msgvault/cmd/documents_vector_test.go b/cmd/msgvault/cmd/documents_vector_test.go index 373eb2c58..81d3bf722 100644 --- a/cmd/msgvault/cmd/documents_vector_test.go +++ b/cmd/msgvault/cmd/documents_vector_test.go @@ -127,7 +127,12 @@ func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { deps := documentsCommandDeps{ openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, } - lane := documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + env := setupEnvironment{ + lookupEnv: func(string) (string, bool) { return "synthetic-key", true }, + consent: setupConsentFromStore(t.Context(), cfg, fixture.Store), + } + lane := documentVectorsLane(cfg, env) + assert.Equal(laneStatePending, lane.State) assert.Equal(map[string]string{"document_embedding": consentMissing, "query_embedding": consentMissing}, lane.ConsentPurposes) assert.ElementsMatch([]string{"msgvault documents vectors consent --yes", "msgvault documents vectors consent --purpose queries --yes"}, lane.Next) for _, purpose := range []string{"documents", "queries"} { @@ -136,19 +141,24 @@ func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { command.SetOut(&output) command.SetArgs([]string{"vectors", "consent", "--purpose", purpose, "--yes"}) require.NoError(command.ExecuteContext(t.Context()), output.String()) - lane = documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + env.consent = setupConsentFromStore(t.Context(), cfg, fixture.Store) + lane = documentVectorsLane(cfg, env) assert.Equal(consentActive, lane.ConsentPurposes["document_embedding"]) if purpose == "documents" { + assert.Equal(laneStatePending, lane.State) assert.Equal(consentMissing, lane.ConsentPurposes["query_embedding"]) assert.Equal([]string{"msgvault documents vectors consent --purpose queries --yes"}, lane.Next) } else { + assert.Equal(laneStateOn, lane.State) assert.Equal(consentActive, lane.ConsentPurposes["query_embedding"]) assert.Equal(consentActive, lane.Consent) assert.Empty(lane.Next) } } cfg.Vector.Embeddings.Endpoint = "https://changed.example.test/v1" - lane = documentVectorsLane(cfg, setupEnvironment{consent: setupConsentFromStore(t.Context(), cfg, fixture.Store)}) + env.consent = setupConsentFromStore(t.Context(), cfg, fixture.Store) + lane = documentVectorsLane(cfg, env) + assert.Equal(laneStatePending, lane.State) assert.Equal(map[string]string{"document_embedding": consentMissing, "query_embedding": consentMissing}, lane.ConsentPurposes) } diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index 4ca7d5f07..dff886384 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -318,6 +318,7 @@ func personSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Reason += "; text search is not ready: " + text.Reason } if lane.Consent != consentActive { + lane.State = laneStatePending lane.Next = []string{"msgvault person provider consent --semantic-embeddings --yes"} } case cfg.Vector.Enabled: @@ -348,6 +349,7 @@ func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { } lane.Reason = "eligible images and short videos are embedded with bounded message context" if lane.Consent != consentActive { + lane.State = laneStatePending lane.Next = []string{"msgvault multimodal build --yes"} } env.reportMissingCredential(&lane, multimodal.APIKeyEnv) @@ -385,6 +387,7 @@ func documentsLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Reason = fmt.Sprintf("region %s; retention=%s, training=%s; uploads are manual-only", documents.Region, documents.RetentionPosture, documents.TrainingPosture) if lane.Consent != consentActive { + lane.State = laneStatePending if env.exists(manifest) { lane.Next = []string{ "msgvault documents consent-mistral --capabilities " + manifest + " --yes", @@ -430,6 +433,9 @@ func documentVectorsLane(cfg *config.Config, env setupEnvironment) laneStatus { "query_embedding": env.consentState(func(s setupConsentState) bool { return s.QueryEmbedding }), } lane.Consent = env.consentState(func(s setupConsentState) bool { return s.DocumentEmbedding && s.QueryEmbedding }) + if lane.Consent != consentActive { + lane.State = laneStatePending + } if lane.ConsentPurposes["document_embedding"] != consentActive { lane.Next = append(lane.Next, "msgvault documents vectors consent --yes") } @@ -462,8 +468,11 @@ func peopleInferenceLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Schedule = "cron " + sweep.Schedule lane.Consent = env.consentState(func(s setupConsentState) bool { return s.PersonInference }) lane.Reason = "runs for tracked people only; deterministic contact state refreshes for everyone through the activity job" - if lane.Consent != consentActive && name != "" { - lane.Next = []string{"msgvault person provider consent " + name + " --yes"} + if lane.Consent != consentActive { + lane.State = laneStatePending + if name != "" { + lane.Next = []string{"msgvault person provider consent " + name + " --yes"} + } } lane.Next = append(lane.Next, "msgvault person track ") if err == nil && provider.Auth != peoplesweep.AuthNone && provider.Credential == peoplesweep.CredentialEnv { diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 659dfa002..7bde6d8ec 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -493,7 +493,8 @@ func planTextSearch(loaded *config.Config, detection setupDetection) setupLanePl Path: []string{tomlTableVector, "embeddings"}, Values: map[string]any{ "api_format": "openai", "endpoint": detection.ollamaEndpoint, - "model": setupOllamaTextModel, "dimension": setupOllamaTextDim, + "api_key_env": "", + "model": setupOllamaTextModel, "dimension": setupOllamaTextDim, "document_prefix": setupOllamaDocPrefix, "query_prefix": setupOllamaQueryPrefix, "max_input_chars": setupOllamaMaxInput, }, diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 386218673..0c9559131 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -328,7 +328,52 @@ func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { output, err = fixture.run(t, "status", "--json") require.NoError(err, output) require.NoError(json.Unmarshal([]byte(output), &report)) - assert.Equal(laneStateOn, findLane(t, report, lane).State) + restored := findLane(t, report, lane) + assert.NotContains(restored.Reason, "not set") + wantState := laneStateOn + if restored.Consent == consentMissing || restored.Consent == consentUnknown { + wantState = laneStatePending + } + assert.Equal(wantState, restored.State) + } +} + +func TestSetupStatusConsentGatedLanesRequireActiveConsent(t *testing.T) { + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + for _, key := range []string{setupVoyageKeyEnv, "MISTRAL_API_KEY", setupOpenAIKeyEnv} { + fixture.env[key] = setupProvidersTestKey + } + fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + require.NoError(t, err, output) + loaded := fixture.load(t) + for _, test := range []struct { + consent *setupConsentState + wantConsent, wantState string + }{ + {wantConsent: consentUnknown, wantState: laneStatePending}, + {consent: &setupConsentState{}, wantConsent: consentMissing, wantState: laneStatePending}, + {consent: &setupConsentState{ + Documents: true, Visual: true, PersonInference: true, PersonSemantic: true, + DocumentEmbedding: true, QueryEmbedding: true, + }, wantConsent: consentActive, wantState: laneStateOn}, + } { + t.Run(test.wantConsent, func(t *testing.T) { + assert := assert.New(t) + report := buildLaneReport(loaded, setupEnvironment{ + lookupEnv: fixture.lookupEnv, fileExists: func(path string) bool { return fixture.files[path] }, + consent: test.consent, + }) + for _, name := range []string{lanePersonSearch, laneVisualSearch, laneDocuments, laneDocumentVectors, lanePeopleInference} { + lane := findLane(t, report, name) + assert.Equal(test.wantConsent, lane.Consent, name) + assert.Equal(test.wantState, lane.State, name) + if test.wantState == laneStatePending { + assert.NotEmpty(lane.Next, name) + } + } + assert.Equal(laneStateOn, findLane(t, report, laneTextSearch).State) + }) } } @@ -533,7 +578,10 @@ func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { func TestSetupProvidersLocalOllamaFallback(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` +[vector.embeddings] +api_key_env = "STALE_PROVIDER_KEY" +`) fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} output, err := fixture.run(t, "providers", "--allow-sensitive") @@ -558,6 +606,11 @@ func TestSetupProvidersLocalOllamaFallback(t *testing.T) { assert.Equal(peoplesweep.CredentialNone, profile.Credential) assert.Equal("http://localhost:11434/v1", profile.Endpoint) assert.Contains(output, "stays on this machine") + output, err = fixture.run(t, "status", "--json") + require.NoError(err, output) + var report laneReport + require.NoError(json.Unmarshal([]byte(output), &report)) + assert.Equal(laneStateOn, findLane(t, report, laneTextSearch).State) } func TestSetupProvidersSkipsRemoteOllama(t *testing.T) { diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index dbf6c39f7..4b106922c 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -44,6 +44,11 @@ Only absent keys receive schedule defaults. Missing embedding credentials leave text search and its dependent people-search and document-vector lanes pending in the status report. +Consent-gated lanes also remain pending until their required consents are +active, including when the consent records cannot be read. Local Ollama +setup clears any old `api_key_env` setting because its selected loopback +endpoint does not require authentication. + For existing text endpoints, setup recognizes the exact OpenAI and Voyage API hosts over HTTPS and loopback servers. Other hosted endpoints are custom: configure their people-search and document-vector lanes explicitly, then From ed3ac6e04528efc7b0164eff5e0dbf185b8a22d1 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 5 Sep 2026 07:33:35 -0500 Subject: [PATCH 07/10] fix(setup): report unavailable vector backends and finish indexing Configured vector lanes cannot run when the binary lacks their database backend. Use the same compiled-backend check for setup and status, and carry the pending state to dependent lanes with rebuild guidance. Consent records authorize document-vector work but do not create an index. Include the build command after both consent steps so the setup follow-ups finish the indexing work. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/setup_lanes.go | 26 +++++++++++++++ cmd/msgvault/cmd/setup_providers.go | 19 ++++------- cmd/msgvault/cmd/setup_providers_test.go | 42 ++++++++++++++++++++++++ docs/usage/recommended-configuration.md | 4 +++ 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index dff886384..1bcc15467 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -20,6 +20,8 @@ import ( "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" vectordocument "go.kenn.io/msgvault/internal/vector/document" + "go.kenn.io/msgvault/internal/vector/pgvector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" ) // Lane and consent states shared by `setup providers` and `setup status`. @@ -147,6 +149,21 @@ func defaultFileExists(path string) bool { return err == nil && info.Mode().IsRegular() } +// Like daemon startup, select the concrete backend from the archive DSN, +// not the declarative vector.backend marker. +func setupVectorBackend(cfg *config.Config) (backend, unavailable string) { + if store.IsPostgresURL(cfg.DatabaseDSN()) { + if !pgvector.Available() { + return "pgvector", "pgvector support is not compiled in; rebuild with `go build -tags \"fts5 sqlite_vec pgvector\" ./cmd/msgvault`, then re-run setup" + } + return "pgvector", "" + } + if !sqlitevec.Available() { + return "sqlite-vec", "sqlite-vec support is not compiled in; rebuild with `make build`, then re-run setup" + } + return "sqlite-vec", "" +} + // setupVoyageManifestPath is the path setup recommends for the Voyage // capability manifest, so a re-run can enable the visual lane once the probe // has written it. @@ -286,6 +303,10 @@ func textSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { } else { lane.Reason = "per-message vectors; no conversation-window context (Voyage contextual only)" } + if _, unavailable := setupVectorBackend(cfg); unavailable != "" { + lane.State = laneStatePending + lane.Reason += "; " + unavailable + } env.reportMissingCredential(&lane, embeddings.APIKeyEnv) return lane } @@ -340,6 +361,11 @@ func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.Model = multimodal.Model lane.Schedule = embedScheduleSummary(multimodal.Schedule) lane.Consent = env.consentState(func(s setupConsentState) bool { return s.Visual }) + if _, unavailable := setupVectorBackend(cfg); unavailable != "" { + lane.State, lane.Reason = laneStatePending, unavailable + env.reportMissingCredential(&lane, multimodal.APIKeyEnv) + return lane + } if !env.exists(multimodal.CapabilitiesFile) { lane.State = laneStatePending lane.Reason = "capabilities_file is missing; the daemon refuses every vector lane until it exists" diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 7bde6d8ec..b0e0e8e18 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -22,9 +22,6 @@ import ( "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/documentindex" "go.kenn.io/msgvault/internal/peoplesweep" - "go.kenn.io/msgvault/internal/store" - "go.kenn.io/msgvault/internal/vector/pgvector" - "go.kenn.io/msgvault/internal/vector/sqlitevec" ) const ( @@ -213,17 +210,9 @@ func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setup voyageKey: env.hasEnv(setupVoyageKeyEnv), mistralKeyEnv: loaded.Attachments.Documents.APIKeyEnv, openAIKey: env.hasEnv(setupOpenAIKeyEnv), - backend: "sqlite-vec", } detection.mistralKey = env.hasEnv(detection.mistralKeyEnv) - if store.IsPostgresURL(loaded.DatabaseDSN()) { - detection.backend = "pgvector" - if !pgvector.Available() { - detection.backendUnavailable = "pgvector support is not compiled in; rebuild with `go build -tags \"fts5 sqlite_vec pgvector\"`, then re-run setup" - } - } else if !sqlitevec.Available() { - detection.backendUnavailable = "sqlite-vec support is not compiled in; rebuild with `make build`, then re-run setup" - } + detection.backend, detection.backendUnavailable = setupVectorBackend(loaded) if path := loaded.Vector.Multimodal.CapabilitiesFile; path != "" && env.exists(path) { detection.voyageManifest = path } else if path := setupVoyageManifestPath(loaded); env.exists(path) { @@ -628,7 +617,11 @@ func planDocumentVectors(loaded *config.Config, plan *setupProvidersPlan) setupL lane.edits = []config.TableEdit{{ Path: []string{"attachments", "documents", "index", "embeddings"}, Values: map[string]any{"enabled": true}, }} - lane.next = []string{"msgvault documents vectors consent --yes", "msgvault documents vectors consent --purpose queries --yes"} + lane.next = []string{ + "msgvault documents vectors consent --yes", + "msgvault documents vectors consent --purpose queries --yes", + "msgvault documents vectors build", + } } return lane } diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 0c9559131..497323c11 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -277,6 +277,40 @@ func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { } } +func TestSetupStatusConfiguredVectorLanesRequireCompiledBackend(t *testing.T) { + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + output, err := fixture.run(t, "providers", "--yes") + require.NoError(t, err, output) + loaded := fixture.load(t) + previous := cfg + cfg = loaded + t.Cleanup(func() { cfg = previous }) + for _, dsn := range []string{"", "postgres://localhost/setup_test"} { + t.Run(dsn, func(t *testing.T) { + assert := assert.New(t) + loaded.Data.DatabaseURL = dsn + report := buildLaneReport(loaded, setupEnvironment{ + lookupEnv: fixture.lookupEnv, fileExists: func(path string) bool { return fixture.files[path] }, + consent: &setupConsentState{Documents: true, Visual: true, PersonSemantic: true, DocumentEmbedding: true, QueryEmbedding: true}, + }) + startupErr := precheckVectorFeatures(loaded.DatabaseDSN()) + for _, name := range []string{laneTextSearch, lanePersonSearch, laneVisualSearch, laneDocumentVectors} { + lane := findLane(t, report, name) + if startupErr != nil { + assert.Equal(laneStatePending, lane.State, name) + assert.Contains(lane.Reason, "not compiled in", name) + assert.Contains(lane.Reason, "rebuild", name) + } else { + assert.Equal(laneStateOn, lane.State, name) + } + } + }) + } +} + func TestSetupProvidersRequiresSensitiveOptIn(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -507,6 +541,14 @@ func TestSetupProvidersMistralEnablesDocumentsAndVectors(t *testing.T) { assert.Contains(output, "documents consent-mistral --capabilities "+manifest+" --yes") assert.Contains(output, "msgvault documents vectors consent --yes") assert.Contains(output, "retention=zdr, training=opted-out") + // The follow-ups must include actual indexing after both consent steps. + build := strings.LastIndex(output, "msgvault documents vectors build") + require.NotEqual(-1, build) + for _, consent := range []string{"msgvault documents vectors consent --yes", "msgvault documents vectors consent --purpose queries --yes"} { + position := strings.LastIndex(output, consent) + require.NotEqual(-1, position) + assert.Less(position, build) + } } func TestSetupProvidersRejectsUnknownDocumentPostures(t *testing.T) { diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index 4b106922c..ef6190b48 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -44,6 +44,9 @@ Only absent keys receive schedule defaults. Missing embedding credentials leave text search and its dependent people-search and document-vector lanes pending in the status report. +Configured vector lanes also stay pending when the binary lacks the backend +required by the archive database. Status includes the rebuild command. + Consent-gated lanes also remain pending until their required consents are active, including when the consent records cannot be read. Local Ollama setup clears any old `api_key_env` setting because its selected loopback @@ -189,6 +192,7 @@ msgvault documents consent-mistral --capabilities ~/.msgvault/mistral-capabiliti msgvault documents build --capabilities ~/.msgvault/mistral-capabilities.json --yes msgvault documents vectors consent --yes # when document vectors are enabled msgvault documents vectors consent --purpose queries --yes +msgvault documents vectors build ``` Document-vector consent covers document text. Query consent separately From 6f98633ff9f2efa2388ed8c0fdca893d4abeaf77 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 5 Sep 2026 08:30:29 -0500 Subject: [PATCH 08/10] fix(setup): match visual consent and complete unknown postures A consented visual generation may belong to an old configuration or capability manifest. Compare both fingerprints before reporting active consent, using the same upload policy calculation as the runtime. The document commands reject unknown postures, so retaining those values leaves setup unable to finish. Treat unknown as unset and let the Mistral confirmation complete those assertions even on an enabled document lane. Preserve known assertions unless their corresponding flags replace them. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/multimodal_probe.go | 39 +++++++++++++ cmd/msgvault/cmd/serve_vector.go | 35 ++---------- .../serve_vector_visual_credentials_test.go | 56 +++++++++++++++++++ cmd/msgvault/cmd/setup_lanes.go | 33 +++++++++-- cmd/msgvault/cmd/setup_providers.go | 4 +- cmd/msgvault/cmd/setup_providers_test.go | 42 ++++++++++++++ docs/usage/recommended-configuration.md | 9 ++- internal/vector/visual/voyage.go | 19 +++++-- 8 files changed, 192 insertions(+), 45 deletions(-) diff --git a/cmd/msgvault/cmd/multimodal_probe.go b/cmd/msgvault/cmd/multimodal_probe.go index 3476cb483..c8a8cb617 100644 --- a/cmd/msgvault/cmd/multimodal_probe.go +++ b/cmd/msgvault/cmd/multimodal_probe.go @@ -12,6 +12,8 @@ import ( "go.kenn.io/msgvault/internal/fileutil" "go.kenn.io/msgvault/internal/providercredentials" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/visual" ) var ( @@ -145,6 +147,43 @@ func writeVisualCapabilityManifest(path string, manifest voyage.CapabilityManife return nil } +// visualVoyageConfig binds the manifest to the media policy used by both +// runtime uploads and setup's read-only consent check. +func visualVoyageConfig(cfg vector.Config) (visual.VoyageConfig, error) { + manifest, err := loadVisualCapabilityManifest(cfg.Multimodal.CapabilitiesFile) + if err != nil { + return visual.VoyageConfig{}, err + } + media := visual.DefaultMediaPolicy() + media.IncludeImages = cfg.Multimodal.ImagesEnabled() || cfg.Multimodal.ImageQueriesEnabled() + media.IncludeVideo = cfg.Multimodal.VideoEnabled() + media.AllowAnimatedGIF = cfg.Multimodal.AnimatedGIFsEnabled() + return visual.VoyageConfig{ + Model: cfg.Multimodal.Model, Dimension: cfg.Multimodal.Dimension, + Manifest: manifest, Media: media, + }, nil +} + +// loadVisualCapabilityManifest reads and strictly validates the operator's +// probed Voyage capability manifest. The multimodal lane cannot run without +// one: nothing has upload authority until a probe recorded it. +func loadVisualCapabilityManifest(path string) (voyage.CapabilityManifest, error) { + if strings.TrimSpace(path) == "" { + return voyage.CapabilityManifest{}, errors.New( + "vector.multimodal.capabilities_file is not set; run `msgvault multimodal probe` and configure the manifest path") + } + file, err := os.Open(path) + if err != nil { + return voyage.CapabilityManifest{}, fmt.Errorf("open Voyage capability manifest: %w", err) + } + defer func() { _ = file.Close() }() + manifest, err := voyage.DecodeCapabilityManifest(file) + if err != nil { + return voyage.CapabilityManifest{}, fmt.Errorf("decode Voyage capability manifest %s: %w", path, err) + } + return manifest, nil +} + func init() { multimodalProbeCmd.Flags().StringVar(&multimodalProbeSeeds, "seeds", "", "Private directory with synthetic WebP and MP4 seeds (primary and contrasting variant of each)") multimodalProbeCmd.Flags().StringVar(&multimodalProbeFixtures, "fixtures", "", "Optional directory to keep the generated fixtures (default: temporary)") diff --git a/cmd/msgvault/cmd/serve_vector.go b/cmd/msgvault/cmd/serve_vector.go index ed42aeafe..ce243d5fd 100644 --- a/cmd/msgvault/cmd/serve_vector.go +++ b/cmd/msgvault/cmd/serve_vector.go @@ -10,7 +10,6 @@ import ( "go.kenn.io/docbank/document/voyage" "log/slog" "net/http" - "os" "path/filepath" "slices" "strconv" @@ -806,7 +805,7 @@ func newVisualRuntime( if err != nil { return nil, err } - manifest, err := loadVisualCapabilityManifest(vecCfg.Multimodal.CapabilitiesFile) + providerConfig, err := visualVoyageConfig(vecCfg) if err != nil { return nil, err } @@ -819,15 +818,9 @@ func newVisualRuntime( // input the search layer permits. Document eligibility (the reconciler's // mediaPolicy) stays unchanged. Configs with images already enabled are // identical, so no existing consent fingerprint moves. - providerMedia := mediaPolicy - if vecCfg.Multimodal.ImageQueriesEnabled() { - providerMedia.IncludeImages = true - } - provider, err := visual.NewVoyageProvider(visual.VoyageConfig{ - APIKey: apiKey, Model: vecCfg.Multimodal.Model, - Dimension: vecCfg.Multimodal.Dimension, Manifest: manifest, Media: providerMedia, - HTTPClient: providerHTTPClientWithoutRedirects(httpClient), - }) + providerConfig.APIKey = apiKey + providerConfig.HTTPClient = providerHTTPClientWithoutRedirects(httpClient) + provider, err := visual.NewVoyageProvider(providerConfig) if err != nil { return nil, err } @@ -910,26 +903,6 @@ func visualScopeCheck(s *store.Store, accounts []string, expected []int64) func( } } -// loadVisualCapabilityManifest reads and strictly validates the operator's -// probed Voyage capability manifest. The multimodal lane cannot run without -// one: nothing has upload authority until a probe recorded it. -func loadVisualCapabilityManifest(path string) (voyage.CapabilityManifest, error) { - if strings.TrimSpace(path) == "" { - return voyage.CapabilityManifest{}, errors.New( - "vector.multimodal.capabilities_file is not set; run `msgvault multimodal probe` and configure the manifest path") - } - file, err := os.Open(path) - if err != nil { - return voyage.CapabilityManifest{}, fmt.Errorf("open Voyage capability manifest: %w", err) - } - defer func() { _ = file.Close() }() - manifest, err := voyage.DecodeCapabilityManifest(file) - if err != nil { - return voyage.CapabilityManifest{}, fmt.Errorf("decode Voyage capability manifest %s: %w", path, err) - } - return manifest, nil -} - // eligibleVisualCapabilities filters probed document capabilities to those // whose interleaved twin is also probed when message context is enabled, so // no eligible owner can produce a request shape the manifest does not cover. diff --git a/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go b/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go index b9427424c..188c2ecd2 100644 --- a/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go +++ b/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go @@ -21,12 +21,68 @@ import ( "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/providercredentials" "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/vector/sqlitevec" "go.kenn.io/msgvault/internal/vector/visual" ) type visualCredentialRoundTripFunc func(*http.Request) (*http.Response, error) +func TestSetupVisualConsentMatchesCurrentGenerationAndManifest(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + c := config.NewDefaultConfig() + c.HomeDir = t.TempDir() + c.Vector.Multimodal.Enabled = true + policy, err := voyage.NewPolicy(voyage.PolicyConfig{ + Model: c.Vector.Multimodal.Model, Dimension: c.Vector.Multimodal.Dimension, + Media: media.Policy{MaxBytes: 20 << 20, MaxPixels: 16_000_000, AllowStill: true, AllowVideo: true}, + }) + require.NoError(err) + manifest, err := voyagetest.SyntheticManifest(policy, voyage.CapabilityQueryText) + require.NoError(err) + c.Vector.Multimodal.CapabilitiesFile = filepath.Join(c.HomeDir, "capabilities.json") + require.NoError(writeVisualCapabilityManifest(c.Vector.Multimodal.CapabilitiesFile, manifest)) + fingerprint, err := policy.Fingerprint(manifest) + require.NoError(err) + st := testutil.NewSQLiteTestStore(t) + generation, err := st.EnsureVisualGeneration(t.Context(), store.VisualGenerationSpec{ + Fingerprint: c.Vector.MultimodalGenerationFingerprint(), + Model: c.Vector.Multimodal.Model, Dimension: c.Vector.Multimodal.Dimension, + }) + require.NoError(err) + require.NoError(st.ConsentVisualGeneration(t.Context(), generation.ID, fingerprint)) + assert.True(setupConsentFromStore(t.Context(), c, st).Visual, "matching building generation") + _, err = st.ActivateVisualGeneration(t.Context(), generation.ID, 0) + require.NoError(err) + assert.True(setupConsentFromStore(t.Context(), c, st).Visual, "matching active generation") + + contextChars := c.Vector.Multimodal.MaxContextChars + c.Vector.Multimodal.MaxContextChars++ + assert.False(setupConsentFromStore(t.Context(), c, st).Visual, "configuration changed") + c.Vector.Multimodal.MaxContextChars = contextChars + changed, err := voyagetest.SyntheticManifest(policy, voyage.CapabilityQueryText, voyage.CapabilityImagePNG) + require.NoError(err) + c.Vector.Multimodal.CapabilitiesFile = filepath.Join(c.HomeDir, "changed-capabilities.json") + require.NoError(writeVisualCapabilityManifest(c.Vector.Multimodal.CapabilitiesFile, changed)) + env := setupEnvironment{ + lookupEnv: func(string) (string, bool) { return "synthetic-key", true }, fileExists: defaultFileExists, + consent: setupConsentFromStore(t.Context(), c, st), + } + lane := visualSearchLane(c, env) + assert.Equal(laneStatePending, lane.State) + assert.Equal([]string{"msgvault multimodal build --yes"}, lane.Next) + changedFingerprint, err := policy.Fingerprint(changed) + require.NoError(err) + vf := &visualFeatures{Archive: st, Generation: generation, PolicyFingerprint: changedFingerprint} + require.ErrorContains(requireVisualConsent(t.Context(), vf), "different capability manifest") + require.NoError(st.ConsentVisualGeneration(t.Context(), generation.ID, changedFingerprint)) + require.NoError(requireVisualConsent(t.Context(), vf)) + assert.True(setupConsentFromStore(t.Context(), c, st).Visual, "new policy consent") + c.Vector.Multimodal.CapabilitiesFile = filepath.Join(c.HomeDir, "missing.json") + assert.False(setupConsentFromStore(t.Context(), c, st).Visual, "manifest cannot be read") +} + func (f visualCredentialRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index 1bcc15467..1fd91406b 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -211,11 +211,7 @@ func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.St if consented, err := st.HasActiveDocumentProviderConsent(ctx); err == nil { state.Documents = consented } - if generation, err := st.ActiveVisualGeneration(ctx); err == nil && generation.Consented { - state.Visual = true - } else if generation, err := st.BuildingVisualGeneration(ctx); err == nil && generation.Consented { - state.Visual = true - } + state.Visual = setupVisualConsent(ctx, cfg, st) if cfg.People.Sweep.Enabled { if profile, err := cfg.People.Sweep.Profile(); err == nil { if active, err := st.HasActivePersonInferenceConsent(ctx, profile.Fingerprint); err == nil { @@ -233,6 +229,33 @@ func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.St return state } +func setupVisualConsent(ctx context.Context, cfg *config.Config, st *store.Store) bool { + if !cfg.Vector.Multimodal.Enabled { + return false + } + providerConfig, err := visualVoyageConfig(cfg.Vector) + if err != nil { + return false + } + policy, err := providerConfig.Policy() + if err != nil { + return false + } + policyFingerprint, err := policy.Fingerprint(providerConfig.Manifest) + if err != nil { + return false + } + fingerprint := cfg.Vector.MultimodalGenerationFingerprint() + for _, read := range []func(context.Context) (store.VisualGeneration, error){st.ActiveVisualGeneration, st.BuildingVisualGeneration} { + generation, err := read(ctx) + if err == nil && generation.Consented && generation.Fingerprint == fingerprint && + generation.ConsentPolicyFingerprint == policyFingerprint { + return true + } + } + return false +} + // embeddingProviderName names the embedding destination for the report. func embeddingProviderName(endpoint string) string { if endpoint == "" { diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index b0e0e8e18..75cc03042 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -561,7 +561,7 @@ func planDocuments(loaded *config.Config, detection setupDetection, options setu lane := setupLanePlan{Lane: laneDocuments, Label: "Document attachments", Provider: documents.Provider, Model: documents.Model} manifest := setupMistralManifestPath(loaded) switch { - case documents.Enabled: + case documents.Enabled && documents.RetentionPosture != documentindex.RetentionUnknown && documents.TrainingPosture != documentindex.TrainingUnknown: lane.Action = planActionKeep lane.Reason = "already enabled" case !detection.mistralKey: @@ -937,7 +937,7 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options } func setupPosture(saved, proposed string, override bool) string { - if saved != "" && !override { + if saved != "" && !strings.EqualFold(strings.TrimSpace(saved), "unknown") && !override { return saved } return proposed diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 497323c11..1aba6ea00 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -209,6 +209,48 @@ training_posture = "opted-out" } } +func TestSetupProvidersResolvesUnknownDocumentPostures(t *testing.T) { + for _, enabled := range []bool{false, true} { + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("enabled=%t/explicit=%t", enabled, explicit), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + savedTraining, wantTraining := documentindex.TrainingUnknown, documentindex.TrainingDefaultOptOut + if explicit { + savedTraining, wantTraining = documentindex.TrainingOptedOut, documentindex.TrainingOptedOut + } + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+fmt.Sprintf(` +[attachments.documents] +enabled = %t +retention_posture = "unknown" +training_posture = %q +`, enabled, savedTraining)) + fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey + args := []string{"providers", "--yes"} + wantRetention := documentindex.RetentionStandard + if explicit { + args = append(args, "--document-retention", documentindex.RetentionZDR) + wantRetention = documentindex.RetentionZDR + } + output, err := fixture.run(t, args...) + require.NoError(err, output) + loaded := fixture.load(t) + assert.True(loaded.Attachments.Documents.Enabled) + assert.Equal(wantRetention, loaded.Attachments.Documents.RetentionPosture) + assert.Equal(wantTraining, loaded.Attachments.Documents.TrainingPosture) + assert.Contains(output, "retention="+wantRetention+", training="+wantTraining) + if explicit { + previous := cfg + cfg = loaded + t.Cleanup(func() { cfg = previous }) + _, _, _, _, err := configuredDocumentProfile(writeCommandCapabilityManifest(t, loaded.Attachments.Documents.MaxPagesPerDocument)) + require.NoError(err) + } + }) + } + } +} + func TestSetupProvidersCustomHostedEndpointNeedsExplicitConfiguration(t *testing.T) { for _, endpoint := range []string{"https://api.openai.com.example.test/v1", "https://localhost.example.test/v1", "https://embeddings.example.test/v1"} { t.Run(endpoint, func(t *testing.T) { diff --git a/docs/usage/recommended-configuration.md b/docs/usage/recommended-configuration.md index ef6190b48..e12cdca23 100644 --- a/docs/usage/recommended-configuration.md +++ b/docs/usage/recommended-configuration.md @@ -36,7 +36,10 @@ When enabling a disabled lane, setup preserves saved retention and training postures. Defaults fill only unset values. Pass `--retention-posture` or `--training-posture` to replace the corresponding people-search posture, or `--document-retention` or `--document-training` for document extraction. -Already enabled lanes remain unchanged. +Already enabled lanes with known postures remain unchanged. `unknown` is +unset, not an assertion: setup fills it with the disclosed default, or the +corresponding explicit flag. This also completes an already-enabled document +lane whose postures are still unknown, under the Mistral confirmation. Setup also preserves each explicit `cron` and `run_after_sync` setting for text and visual embeddings, including `cron = ""` and `run_after_sync = false`. @@ -48,7 +51,9 @@ Configured vector lanes also stay pending when the binary lacks the backend required by the archive database. Status includes the rebuild command. Consent-gated lanes also remain pending until their required consents are -active, including when the consent records cannot be read. Local Ollama +active, including when the consent records cannot be read. Visual consent +must match both the current configuration and the capability-manifest policy; +after either changes, run `msgvault multimodal build --yes` again. Local Ollama setup clears any old `api_key_env` setting because its selected loopback endpoint does not require authentication. diff --git a/internal/vector/visual/voyage.go b/internal/vector/visual/voyage.go index e7c17eb65..4defcd679 100644 --- a/internal/vector/visual/voyage.go +++ b/internal/vector/visual/voyage.go @@ -47,12 +47,9 @@ type VoyageProvider struct { // from the manifest. Construction succeeds with zero authorized capabilities; // requests then fail closed per input. func NewVoyageProvider(config VoyageConfig) (*VoyageProvider, error) { - policy, err := voyage.NewPolicy(voyage.PolicyConfig{ - Model: config.Model, Dimension: config.Dimension, - Media: config.Media.documentPolicy(), - }) + policy, err := config.Policy() if err != nil { - return nil, fmt.Errorf("voyage policy: %w", err) + return nil, err } client, err := voyage.NewClient(policy, voyage.ClientConfig{ APIKey: config.APIKey, Timeout: config.Timeout, HTTPClient: config.HTTPClient, @@ -80,6 +77,18 @@ func NewVoyageProvider(config VoyageConfig) (*VoyageProvider, error) { return provider, nil } +// Policy derives the upload policy without resolving credentials or creating +// a transport, so readiness checks use the same identity as the provider. +func (c VoyageConfig) Policy() (voyage.Policy, error) { + policy, err := voyage.NewPolicy(voyage.PolicyConfig{ + Model: c.Model, Dimension: c.Dimension, Media: c.Media.documentPolicy(), + }) + if err != nil { + return voyage.Policy{}, fmt.Errorf("voyage policy: %w", err) + } + return policy, nil +} + // AuthorizedCapabilities returns the sorted capability IDs with probed upload // authority; eligibility policy folds them into rejection revisions. func (p *VoyageProvider) AuthorizedCapabilities() []string { From 4c57c09956c84add8976724d0c8274bc5c114257 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 5 Sep 2026 09:08:33 -0500 Subject: [PATCH 09/10] fix(setup): validate manifests and match document provider consent An existing manifest does not make visual search usable. Apply the runtime policy checks before enabling the lane, including text-query authority, and report invalid manifests as pending. Keep an explicit manifest path even when it is missing so probe guidance respects the operator's choice. Document consent from an older provider configuration must not make the current lane appear ready. Match the configured provider, endpoint, region, model, and postures against active consented profiles without requiring the original probe file to remain at setup's default location. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/multimodal_probe.go | 14 ++- cmd/msgvault/cmd/serve_vector.go | 6 - cmd/msgvault/cmd/setup_lanes.go | 41 ++++++- cmd/msgvault/cmd/setup_providers.go | 35 +++--- cmd/msgvault/cmd/setup_providers_test.go | 143 ++++++++++++++++++++++- internal/store/document_index.go | 25 ++++ 6 files changed, 227 insertions(+), 37 deletions(-) diff --git a/cmd/msgvault/cmd/multimodal_probe.go b/cmd/msgvault/cmd/multimodal_probe.go index c8a8cb617..edadfaecb 100644 --- a/cmd/msgvault/cmd/multimodal_probe.go +++ b/cmd/msgvault/cmd/multimodal_probe.go @@ -158,10 +158,20 @@ func visualVoyageConfig(cfg vector.Config) (visual.VoyageConfig, error) { media.IncludeImages = cfg.Multimodal.ImagesEnabled() || cfg.Multimodal.ImageQueriesEnabled() media.IncludeVideo = cfg.Multimodal.VideoEnabled() media.AllowAnimatedGIF = cfg.Multimodal.AnimatedGIFsEnabled() - return visual.VoyageConfig{ + provider := visual.VoyageConfig{ Model: cfg.Multimodal.Model, Dimension: cfg.Multimodal.Dimension, Manifest: manifest, Media: media, - }, nil + } + policy, err := provider.Policy() + if err != nil { + return visual.VoyageConfig{}, fmt.Errorf("configure visual capability policy: %w", err) + } + // Every visual search embeds its text query through this provider. A + // manifest that only authorizes indexing cannot make the lane usable. + if _, err := policy.Authorize(manifest, voyage.CapabilityQueryText); err != nil { + return visual.VoyageConfig{}, fmt.Errorf("capability manifest does not authorize text queries; re-run `msgvault multimodal probe`: %w", err) + } + return provider, nil } // loadVisualCapabilityManifest reads and strictly validates the operator's diff --git a/cmd/msgvault/cmd/serve_vector.go b/cmd/msgvault/cmd/serve_vector.go index ce243d5fd..26788db6d 100644 --- a/cmd/msgvault/cmd/serve_vector.go +++ b/cmd/msgvault/cmd/serve_vector.go @@ -828,12 +828,6 @@ func newVisualRuntime( // shapes this archive sends: the document capability always, and its // interleaved twin because owning-message context accompanies media // whenever the message has any. - // Every visual search embeds its text query through the same client; - // without probed text-query authority the lane would index (and bill) - // while rejecting every search. Fail initialization with the remedy. - if !slices.Contains(provider.AuthorizedCapabilities(), voyage.CapabilityQueryText) { - return nil, errors.New("the capability manifest does not authorize text queries; re-run `msgvault multimodal probe` and configure the new manifest") - } mediaPolicy.AuthorizedCapabilities = eligibleVisualCapabilities( provider.AuthorizedCapabilities(), vecCfg.Multimodal.MaxContextChars > 0) consumerKey := "visual/" + fingerprint diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index 1fd91406b..c06f9096c 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -168,9 +168,29 @@ func setupVectorBackend(cfg *config.Config) (backend, unavailable string) { // capability manifest, so a re-run can enable the visual lane once the probe // has written it. func setupVoyageManifestPath(cfg *config.Config) string { + if cfg.Vector.Multimodal.CapabilitiesFile != "" { + return cfg.Vector.Multimodal.CapabilitiesFile + } return filepath.Join(cfg.HomeDir, setupVoyageManifestName) } +func setupVisualManifestError(cfg *config.Config, env setupEnvironment) error { + path := setupVoyageManifestPath(cfg) + if !env.exists(path) { + return fmt.Errorf("capability manifest is missing at %s", path) + } + vectorConfig := cfg.Vector + if !vectorConfig.Multimodal.Enabled { + vectorConfig.Multimodal.CapabilitiesFile = path + vectorConfig.Multimodal.Enabled = true + } + _, err := visualVoyageConfig(vectorConfig) + if err != nil { + return fmt.Errorf("invalid visual capability manifest: %w; move the existing manifest aside before probing again", err) + } + return nil +} + // setupMistralManifestPath is the recommended Mistral capability manifest path. func setupMistralManifestPath(cfg *config.Config) string { return filepath.Join(cfg.HomeDir, setupMistralManifestName) @@ -208,8 +228,16 @@ func setupConsentFromStore(ctx context.Context, cfg *config.Config, st *store.St } } } - if consented, err := st.HasActiveDocumentProviderConsent(ctx); err == nil { - state.Documents = consented + documents := cfg.Attachments.Documents + if documents.Enabled && documents.Validate() == nil { + if policy, err := documents.MistralPolicy(); err == nil { + values := policy.Values() + consented, err := st.HasMatchingDocumentProviderConsent(ctx, store.DocumentExtractionProfile{ + Provider: values.Provider, Endpoint: values.Endpoint, Region: values.Region, + Model: values.Model, RetentionPosture: values.Retention, TrainingPosture: values.Training, + }) + state.Documents = err == nil && consented + } } state.Visual = setupVisualConsent(ctx, cfg, st) if cfg.People.Sweep.Enabled { @@ -389,9 +417,9 @@ func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { env.reportMissingCredential(&lane, multimodal.APIKeyEnv) return lane } - if !env.exists(multimodal.CapabilitiesFile) { + if err := setupVisualManifestError(cfg, env); err != nil { lane.State = laneStatePending - lane.Reason = "capabilities_file is missing; the daemon refuses every vector lane until it exists" + lane.Reason = err.Error() + "; the daemon refuses every vector lane until a valid manifest is configured" lane.Next = []string{visualProbeCommand(cfg)} env.reportMissingCredential(&lane, multimodal.APIKeyEnv) return lane @@ -410,12 +438,13 @@ func visualSearchLane(cfg *config.Config, env setupEnvironment) laneStatus { return lane } lane.State = laneStatePending - if env.exists(setupVoyageManifestPath(cfg)) { + err := setupVisualManifestError(cfg, env) + if err == nil { lane.Reason = "probe manifest found; setup can enable the lane" lane.Next = []string{"msgvault setup providers"} return lane } - lane.Reason = "key present; the provider probe needs private synthetic WebP and MP4 seeds before uploads are authorized" + lane.Reason = err.Error() + "; the provider probe needs private synthetic WebP and MP4 seeds before uploads are authorized" lane.Next = []string{visualProbeCommand(cfg), "msgvault setup providers"} return lane } diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index 75cc03042..8c0a13c81 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -190,18 +190,19 @@ func (r ollamaProbeResult) hasModel(name string) bool { // the local Ollama server offers, which probe manifests are already written, // and which vector backend the archive selects. type setupDetection struct { - configKeys toml.MetaData - voyageKey bool - mistralKey bool - mistralKeyEnv string - openAIKey bool - ollama ollamaProbeResult - ollamaEndpoint string - ollamaLoopback bool - voyageManifest string - mistralManifest string - backend string - backendUnavailable string + configKeys toml.MetaData + voyageKey bool + mistralKey bool + mistralKeyEnv string + openAIKey bool + ollama ollamaProbeResult + ollamaEndpoint string + ollamaLoopback bool + voyageManifest string + voyageManifestError string + mistralManifest string + backend string + backendUnavailable string } func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setupProvidersDeps) setupDetection { @@ -213,10 +214,10 @@ func detectSetupProviders(ctx context.Context, loaded *config.Config, deps setup } detection.mistralKey = env.hasEnv(detection.mistralKeyEnv) detection.backend, detection.backendUnavailable = setupVectorBackend(loaded) - if path := loaded.Vector.Multimodal.CapabilitiesFile; path != "" && env.exists(path) { - detection.voyageManifest = path - } else if path := setupVoyageManifestPath(loaded); env.exists(path) { - detection.voyageManifest = path + if err := setupVisualManifestError(loaded, env); err != nil { + detection.voyageManifestError = err.Error() + } else { + detection.voyageManifest = setupVoyageManifestPath(loaded) } if path := setupMistralManifestPath(loaded); env.exists(path) { detection.mistralManifest = path @@ -549,7 +550,7 @@ func planVisualSearch(loaded *config.Config, detection setupDetection) setupLane lane.next = []string{"msgvault multimodal build --yes"} default: lane.Action = planActionPending - lane.Reason = "the provider probe needs private synthetic WebP and MP4 seeds; the lane stays off until the manifest exists" + lane.Reason = detection.voyageManifestError + "; the provider probe needs private synthetic WebP and MP4 seeds; the lane stays off until a valid manifest is configured" lane.edits = setupScheduleEdits(detection.configKeys, "multimodal") lane.next = []string{visualProbeCommand(loaded), "msgvault setup providers"} } diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 1aba6ea00..e4a9b214d 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -16,6 +16,8 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/docbank/document/voyage" + "go.kenn.io/docbank/document/voyage/voyagetest" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/documentindex" "go.kenn.io/msgvault/internal/peoplesweep" @@ -23,6 +25,7 @@ import ( "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/vector" "go.kenn.io/msgvault/internal/vector/pgvector" + "go.kenn.io/msgvault/internal/vector/visual" ) const setupProvidersTestKey = "setup-providers-test-key" @@ -77,6 +80,132 @@ func (f *setupProvidersFixture) lookupEnv(name string) (string, bool) { return value, ok } +func (f *setupProvidersFixture) writeVisualManifest(t *testing.T, path string, capabilities ...string) { + t.Helper() + require := require.New(t) + multimodal := f.load(t).Vector.Multimodal + provider := visual.VoyageConfig{Model: multimodal.Model, Dimension: multimodal.Dimension, Media: visual.DefaultMediaPolicy()} + policy, err := provider.Policy() + require.NoError(err) + manifest, err := voyagetest.SyntheticManifest(policy, capabilities...) + require.NoError(err) + require.NoError(writeVisualCapabilityManifest(path, manifest)) + f.files[path] = true +} + +func TestSetupProvidersRejectsUnusableVisualManifest(t *testing.T) { + for _, kind := range []string{"malformed", "no text queries", "wrong model"} { + t.Run(kind, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + path := filepath.Join(fixture.dir, setupVoyageManifestName) + if kind == "malformed" { + require.NoError(os.WriteFile(path, []byte("not JSON"), 0o600)) + fixture.files[path] = true + } else { + capability := voyage.CapabilityQueryText + if kind == "no text queries" { + capability = voyage.CapabilityImagePNG + } + fixture.writeVisualManifest(t, path, capability) + if kind == "wrong model" { + snapshot, err := config.ReadConfigFile(fixture.path) + require.NoError(err) + _, err = config.EditConfigTables(fixture.path, snapshot.ETag, []config.TableEdit{{ + Path: []string{"vector", "multimodal"}, Values: map[string]any{"model": "different-model"}, + }}) + require.NoError(err) + } + } + output, err := fixture.run(t, "providers", "--yes") + require.NoError(err, output) + loaded := fixture.load(t) + assert.False(loaded.Vector.Multimodal.Enabled) + assert.Contains(output, "capability manifest") + if kind == "no text queries" { + assert.Contains(output, "does not authorize text queries") + } + loaded.Vector.Multimodal.Enabled = true + loaded.Vector.Multimodal.CapabilitiesFile = path + lane := visualSearchLane(loaded, setupEnvironment{ + lookupEnv: fixture.lookupEnv, fileExists: defaultFileExists, consent: &setupConsentState{Visual: true}, + }) + assert.Equal(laneStatePending, lane.State) + assert.Contains(lane.Reason, "capability manifest") + }) + } +} + +func TestSetupProvidersPreservesMissingCustomVisualManifest(t *testing.T) { + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` +[vector.multimodal] +capabilities_file = "{{DIR}}/custom-voyage.json" +`) + fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) + output, err := fixture.run(t, "providers", "--yes") + require.NoError(t, err, output) + loaded := fixture.load(t) + custom := filepath.Join(fixture.dir, "custom-voyage.json") + assert.False(loaded.Vector.Multimodal.Enabled) + assert.Equal(custom, loaded.Vector.Multimodal.CapabilitiesFile) + assert.Contains(output, "--out "+custom+" --yes") + for _, enabled := range []bool{false, true} { + loaded.Vector.Multimodal.Enabled = enabled + lane := visualSearchLane(loaded, setupEnvironment{lookupEnv: fixture.lookupEnv, fileExists: defaultFileExists}) + assert.Equal(laneStatePending, lane.State) + assert.Contains(lane.Next, "msgvault multimodal probe --seeds --out "+custom+" --yes") + } +} + +func TestSetupDocumentConsentMatchesConfiguredProvider(t *testing.T) { + require := require.New(t) + c := config.NewDefaultConfig() + c.Attachments.Documents.Enabled = true + c.Attachments.Documents.RetentionPosture = documentindex.RetentionZDR + c.Attachments.Documents.TrainingPosture = documentindex.TrainingOptedOut + _, profile, err := documentProfileForConfig(&c.Attachments.Documents, commandCapabilityManifest(t, c.Attachments.Documents.MaxPagesPerDocument)) + require.NoError(err) + st := testutil.NewSQLiteTestStore(t) + _, err = st.EnsureDocumentExtractionProfile(t.Context(), profile) + require.NoError(err) + assert.False(t, setupConsentFromStore(t.Context(), c, st).Documents) + require.NoError(st.RecordDocumentProviderConsent(t.Context(), store.DocumentProviderConsent{ + ProfileID: profile.ID, ProfileFingerprint: profile.Fingerprint, + RetentionPosture: profile.RetentionPosture, TrainingPosture: profile.TrainingPosture, + })) + assert.True(t, setupConsentFromStore(t.Context(), c, st).Documents) + for _, field := range []string{"provider", "model", "region", "retention", "training"} { + t.Run(field, func(t *testing.T) { + changed := *c + documents := &changed.Attachments.Documents + switch field { + case "provider": + documents.Provider = "other" + case "model": + documents.Model = "other-model" + case "region": + documents.Region = "other-region" + case "retention": + documents.RetentionPosture = documentindex.RetentionStandard + case "training": + documents.TrainingPosture = documentindex.TrainingDefaultOptOut + } + consent := setupConsentFromStore(t.Context(), &changed, st) + assert.False(t, consent.Documents) + lane := documentsLane(&changed, setupEnvironment{consent: consent, lookupEnv: func(string) (string, bool) { return setupProvidersTestKey, true }}) + assert.Equal(t, laneStatePending, lane.State) + assert.Equal(t, consentMissing, lane.Consent) + }) + } + _, err = st.RetireDocumentExtractionProfile(t.Context(), profile.ID) + require.NoError(err) + assert.False(t, setupConsentFromStore(t.Context(), c, st).Documents) +} + func (f *setupProvidersFixture) personProviderDeps(t *testing.T) personProviderCommandDeps { t.Helper() loaded := f.load(t) @@ -300,7 +429,7 @@ func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { require := require.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+`database_url = "postgres://localhost/setup_test"`) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey - fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes", "--json") require.NoError(err, output) loaded := fixture.load(t) @@ -323,7 +452,7 @@ func TestSetupStatusConfiguredVectorLanesRequireCompiledBackend(t *testing.T) { fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey - fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes") require.NoError(t, err, output) loaded := fixture.load(t) @@ -384,7 +513,7 @@ func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { for _, key := range []string{setupVoyageKeyEnv, "MISTRAL_API_KEY", setupOpenAIKeyEnv} { fixture.env[key] = setupProvidersTestKey } - fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) _, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(err) for lane, key := range map[string]string{ @@ -419,7 +548,7 @@ func TestSetupStatusConsentGatedLanesRequireActiveConsent(t *testing.T) { for _, key := range []string{setupVoyageKeyEnv, "MISTRAL_API_KEY", setupOpenAIKeyEnv} { fixture.env[key] = setupProvidersTestKey } - fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = true + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") require.NoError(t, err, output) loaded := fixture.load(t) @@ -472,7 +601,9 @@ func TestSetupProvidersPreservesExplicitSchedules(t *testing.T) { "\n[vector.embed.schedule]\n"+schedule.toml+ "\n[vector.multimodal.schedule]\n"+schedule.toml+"\n") fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey - fixture.files[filepath.Join(fixture.dir, setupVoyageManifestName)] = manifest + if manifest { + fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) + } output, err := fixture.run(t, "providers", "--yes") require.NoError(t, err, output) loaded := fixture.load(t) @@ -547,7 +678,7 @@ func TestSetupProvidersEnablesVisualLaneWhenManifestExists(t *testing.T) { fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey manifest := filepath.Join(fixture.dir, setupVoyageManifestName) - fixture.files[manifest] = true + fixture.writeVisualManifest(t, manifest, voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes") require.NoError(err, output) diff --git a/internal/store/document_index.go b/internal/store/document_index.go index 67ee55698..9adae7b0e 100644 --- a/internal/store/document_index.go +++ b/internal/store/document_index.go @@ -477,6 +477,31 @@ func (s *Store) HasActiveDocumentProviderConsent(ctx context.Context) (bool, err return consented, nil } +// HasMatchingDocumentProviderConsent reports consent for an enabled, unretired +// profile with the requested provider, endpoint, region, model, and postures. +// Unlike the journal bootstrap gate, setup must not borrow consent from an +// unrelated provider policy. The profile's ID and content policy are not used. +func (s *Store) HasMatchingDocumentProviderConsent(ctx context.Context, profile DocumentExtractionProfile) (bool, error) { + var consented bool + err := s.db.QueryRowContext(ctx, s.dialect.Rebind(` + SELECT EXISTS ( + SELECT 1 + FROM document_extraction_profiles p + JOIN document_provider_consents c ON c.profile_id = p.id + WHERE p.enabled = TRUE AND p.retired_at IS NULL + AND c.profile_fingerprint = p.fingerprint + AND c.retention_posture = p.retention_posture + AND c.training_posture = p.training_posture + AND p.provider = ? AND p.endpoint = ? AND p.region = ? + AND p.model = ? AND p.retention_posture = ? AND p.training_posture = ? + )`), profile.Provider, profile.Endpoint, profile.Region, profile.Model, + profile.RetentionPosture, profile.TrainingPosture).Scan(&consented) + if err != nil { + return false, fmt.Errorf("read matching document provider consent: %w", err) + } + return consented, nil +} + func (s *Store) GetDocumentIndexStatus(ctx context.Context, profileID string) (DocumentIndexStatus, error) { if profileID == "" { return DocumentIndexStatus{}, errors.New("document index status requires a profile ID") From c46dbd17aad2796758ca8b8b5d51d52e62b73f4b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 5 Sep 2026 12:29:47 -0500 Subject: [PATCH 10/10] test(setup): preserve configured path spelling on Windows The custom-manifest fixture writes forward slashes into TOML on every platform. Setup preserves that spelling, but the test expected native Windows separators and failed the Windows CLI shard. Match the fixture's path spelling while retaining the checks that setup preserves the custom path and uses it in probe guidance. Generated with Codex Co-authored-by: Codex --- cmd/msgvault/cmd/setup_providers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index e4a9b214d..d729461fc 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -149,7 +149,7 @@ capabilities_file = "{{DIR}}/custom-voyage.json" output, err := fixture.run(t, "providers", "--yes") require.NoError(t, err, output) loaded := fixture.load(t) - custom := filepath.Join(fixture.dir, "custom-voyage.json") + custom := filepath.ToSlash(filepath.Join(fixture.dir, "custom-voyage.json")) assert.False(loaded.Vector.Multimodal.Enabled) assert.Equal(custom, loaded.Vector.Multimodal.CapabilitiesFile) assert.Contains(output, "--out "+custom+" --yes")