diff --git a/.roborev.toml b/.roborev.toml index 7cfbdd69fe..101e3b9b32 100644 --- a/.roborev.toml +++ b/.roborev.toml @@ -1,10 +1,12 @@ review_guidelines = """ -agentsview is a single-user developer tool. Default mode binds to +agentsview defaults to a single-user developer tool. Local mode binds to 127.0.0.1. Optional managed Caddy proxy mode allows LAN access while keeping the backend on loopback. Optional remote access mode binds to 0.0.0.0 with bearer-token auth for use over secure tunnels (Tailscale, -SSH, reverse proxy with TLS). Not designed for multi-user or -internet-facing deployment. +SSH, reverse proxy with TLS). These legacy modes are not designed for +request-multiplexed multi-user deployment. Opt-in hosted raw processing has +a separate threat model: one configured tenant, schema, restricted role and +server instance, with authenticated uploads treated as untrusted input. Key assumptions reviewers MUST account for: @@ -15,7 +17,9 @@ Key assumptions reviewers MUST account for: bearer token is required for all API requests (including localhost, to prevent bypass via reverse proxy). An auth token is auto-generated at startup if missing. Do not flag missing auth on - local-only code paths. DO flag any path that lets the backend bind + local-only code paths. Hosted mode always requires authentication; + raw-sync routes use device credentials and scoped tokens separately + from the viewer bearer token. DO flag any path that lets the backend bind non-loopback in proxy mode, or missing subnet checks for non-loopback Caddy binds. @@ -37,8 +41,9 @@ Key assumptions reviewers MUST account for: 4. XSS: {@html renderMarkdown(...)} is safe — renderMarkdown() sanitizes via DOMPurify before returning HTML. -5. RATE LIMITING: Single-user tool. Do not flag missing rate limits - or concurrency caps. +5. RATE LIMITING: Do not flag missing rate limits or concurrency + caps in local/legacy single-user paths. Hosted raw ingestion and + derivation must bound body, materialization, output and worker resources. 6. CORS: corsMiddleware requires matching Origin for mutating requests. Allowed origins = loopback variants + public_url + @@ -46,25 +51,31 @@ Key assumptions reviewers MUST account for: token) allow the request origin. Do not flag as overly permissive unless origins outside the configured set are accepted. -7. INPUT VALIDATION: Body size limits not required — backend is - loopback-only; in proxy mode Caddy subnet filtering limits +7. INPUT VALIDATION: In local/legacy paths, body size limits are not + required — backend is loopback-only; in proxy mode Caddy subnet filtering limits clients to trusted hosts. In remote mode, bearer token gates - all API access. + all API access. Hosted raw authentication does not make source bytes + trusted; review bounded protocol and parser validation. 8. SESSION DATA: Displaying session contents (tool args, commands, paths) is the tool's purpose. The user owns these files. Do not - flag as sensitive data exposure. + flag as sensitive data exposure within that user's local/legacy access + boundary. Hosted reads must preserve tenant and public-identity isolation. -9. SUBPROCESS ENV: Agent CLI subprocesses intentionally inherit the - parent environment. Do not flag env var inheritance. +9. SUBPROCESS ENV: Local insight/capture agent CLIs intentionally inherit + the parent environment. Hosted raw parser children require an explicit + minimal environment and pre-runtime inherited-descriptor closure. -10. SESSION PARSING: Input files are from local agent CLIs, not - adversarial. Do not flag missing cycle detection, recursion - limits, or unreachable-node checks in DAG traversals. +10. SESSION PARSING: Local/legacy input files are user-owned. Do not flag + missing cycle detection, recursion limits or unreachable-node checks + there. Hosted raw input is untrusted; review fail-closed filesystem, + network, process and resource isolation before parsing. -11. WRITE ATOMICITY: Per-session transactions only. Full resync - recovers partial state. Do not flag non-atomic multi-session - writes. +11. WRITE ATOMICITY: Local/legacy writes use per-session transactions; + full resync recovers partial state. Hosted publication must atomically + fence selection and lease generations, normalized rows, provenance, + public identity, curation, outbox and job outcome. Partial sources must + retain unresolved proof and retry finitely. 12. TOCTOU ON LOCAL FILES: Files in ~/.agentsview/ are user-owned. An attacker with home-directory access already has equivalent @@ -132,7 +143,11 @@ Key assumptions reviewers MUST account for: go.mod toolchain version before flagging unfamiliar language features. -Do NOT flag issues that only apply to public-facing, multi-tenant, -or internet-exposed services. Focus on bugs, logic errors, data -corruption risks, and code quality issues. +For local/legacy modes, do not impose requirements that apply only to +public-facing multi-tenant services. For opt-in hosted raw processing, +review its actual tenant/role/catalog boundary, immutable source provenance, +source-selection and lease fences, bounded isolated parsing, public-ID +ambiguity, curation survival and partial-source retry semantics. Do not +assume request-multiplexed tenancy. Focus on bugs, data corruption, boundary +violations and maintainability within each mode's supported contract. """ diff --git a/README.md b/README.md index b0e1bfeab0..df35d71d3b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +--- +last_edited: 2026-09-11 +--- + # agentsview Browse, search, and track costs across all your AI coding agents. One binary, no @@ -557,6 +561,11 @@ agentsview pg status --all # show status for every configured PG target agentsview pg serve # serve web UI from the default PG target (read-only) ``` +Opt-in [hosted raw processing](docs/hosted-raw-sync.md) lets `pg serve` parse +uploaded sources directly into PostgreSQL. It requires explicit tenant/schema +provisioning, a restricted runtime role, authentication and Linux isolation. +Ordinary PG serving remains read-only; `pg push` refuses hosted-owned schemas. + Single-target configs still use the legacy `[pg]` block. To manage more than one PostgreSQL destination, define named `[pg.NAME]` blocks and set `default_pg` when more than one target exists: @@ -576,7 +585,9 @@ exclude_projects = ["scratch"] Named target names are normalized case-insensitively. `all`, `local`, and the legacy `[pg]` field names `url`, `schema`, `machine_name`, `allow_insecure`, -`projects`, and `exclude_projects` cannot be used for `[pg.NAME]`. +`projects`, `exclude_projects`, `raw_tenant`, `raw_derivation`, +`raw_poll_seconds`, `raw_attempt_seconds`, and `raw_max_attempts` cannot be used +for `[pg.NAME]`. `AGENTSVIEW_PG_URL`, `AGENTSVIEW_PG_SCHEMA`, and `AGENTSVIEW_PG_MACHINE` still work, but in named-target mode they apply only to the effective default target. diff --git a/cmd/agentsview/cli.go b/cmd/agentsview/cli.go index 51bf5055f5..1a39653052 100644 --- a/cmd/agentsview/cli.go +++ b/cmd/agentsview/cli.go @@ -651,6 +651,8 @@ func newPGCommand() *cobra.Command { cmd.AddCommand(newPGServeCommand()) cmd.AddCommand(newPGVectorsCommand()) cmd.AddCommand(newPGServiceCommand()) + cmd.AddCommand(newPGHostedProvisionCommand()) + cmd.AddCommand(newPGRawReparseCommand()) return cmd } diff --git a/cmd/agentsview/main.go b/cmd/agentsview/main.go index 9eb9fcd1c3..0329d35308 100644 --- a/cmd/agentsview/main.go +++ b/cmd/agentsview/main.go @@ -24,6 +24,7 @@ import ( "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" "go.kenn.io/agentsview/internal/recall/extract" "go.kenn.io/agentsview/internal/remotesync" "go.kenn.io/agentsview/internal/secrets" @@ -60,6 +61,9 @@ const ( ) func main() { + if handled, code := rawderive.RunParserChild(os.Args[1:]); handled { + os.Exit(code) + } // Turn on the agentsview-test-fixture deny-list before any scan // runs. The secrets package keeps the filter off by default so unit // tests in this repo (which use the same random-looking fixtures diff --git a/cmd/agentsview/pg.go b/cmd/agentsview/pg.go index 02ce875a42..6639519811 100644 --- a/cmd/agentsview/pg.go +++ b/cmd/agentsview/pg.go @@ -114,6 +114,11 @@ func runPGPush( return err } + for _, target := range targets { + if target.PG.RawTenant != "" || target.PG.RawDerivation { + return errors.New("pg push cannot mutate a hosted-owned projection") + } + } applyClassifierConfig(appCfg) ctx, stop := signal.NotifyContext( context.Background(), os.Interrupt, @@ -172,6 +177,9 @@ func runPGPushTarget( if err != nil { return err } + if target.PG.RawTenant != "" || target.PG.RawDerivation { + return errors.New("pg push cannot mutate a hosted-owned projection") + } if target.PG.URL == "" { return fmt.Errorf("url not configured") } @@ -479,11 +487,12 @@ func loadPGServeConfig(cmd *cobra.Command) (config.Config, string, error) { } type pgServeStartup struct { - cfg config.Config - ctx context.Context - rtOpts serveRuntimeOptions - srv *server.Server - cleanup func() + cfg config.Config + ctx context.Context + rtOpts serveRuntimeOptions + srv *server.Server + cleanup func() + startWorker func() } var preparePGServe = preparePGServeImpl @@ -501,6 +510,12 @@ func preparePGServeImpl(appCfg config.Config, basePath string) (pgServeStartup, return pgServeStartup{}, errors.New("pg serve: url not configured") } + if err := pgCfg.ValidateRawDerivation(appCfg.RequireAuth); err != nil { + return pgServeStartup{}, err + } + if pgCfg.RawTenant != "" { + return prepareHostedPGServe(appCfg, pgCfg, basePath) + } applyClassifierConfig(appCfg) store, err := postgres.NewStore( pgCfg.URL, pgCfg.Schema, pgCfg.AllowInsecure, @@ -632,8 +647,16 @@ func runPGServe(appCfg config.Config, basePath string) { if err != nil { fatal("%v", err) } + if err = runPreparedPGServe(startup); err != nil { + fatal("%v", err) + } +} + +// Return through cleanup before the outer CLI may call os.Exit, including +// readiness failures and unexpected server/proxy exits. +func runPreparedPGServe(startup pgServeStartup) error { defer startup.cleanup() - appCfg = startup.cfg + appCfg := startup.cfg ctx := startup.ctx rtOpts := startup.rtOpts srv := startup.srv @@ -646,11 +669,14 @@ func runPGServe(appCfg config.Config, basePath string) { ) if err != nil { if errors.Is(err, context.Canceled) { - return + return nil } - fatal("pg serve: %v", err) + return fmt.Errorf("pg serve: %w", err) } + if startup.startWorker != nil { + startup.startWorker() + } // Write the kit runtime record so CLI commands can discover this // daemon. ReadOnly=true marks it as pg serve (read-only) // so clients can select an appropriate transport. @@ -677,8 +703,9 @@ func runPGServe(appCfg config.Config, basePath string) { } if err := waitForServerRuntime(ctx, srv, rt); err != nil { - fatal("pg serve: %v", err) + return fmt.Errorf("pg serve: %w", err) } + return nil } func writePGServeRuntimeRecord(rt *serveRuntime) bool { diff --git a/cmd/agentsview/pg_raw_derive.go b/cmd/agentsview/pg_raw_derive.go new file mode 100644 index 0000000000..e2a239b7c6 --- /dev/null +++ b/cmd/agentsview/pg_raw_derive.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/postgres" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" + "go.kenn.io/agentsview/internal/server" +) + +func rawProcessingVersion() string { return fmt.Sprintf("parser-data-%d", db.CurrentDataVersion()) } + +// pgRawRuntime owns one bounded sequential worker/maintenance loop. Stop joins +// all materialization/parser work before custody and database owners may close. +type pgRawRuntime struct { + mu sync.Mutex + stopped, started bool + cancel context.CancelFunc + done chan struct{} + ctx context.Context + batch func(context.Context) + interval time.Duration +} + +func newPGRawRuntime(parent context.Context, interval time.Duration, batch func(context.Context)) *pgRawRuntime { + ctx, cancel := context.WithCancel(parent) + return &pgRawRuntime{ctx: ctx, cancel: cancel, done: make(chan struct{}), batch: batch, interval: interval} +} +func (r *pgRawRuntime) Start() { + r.mu.Lock() + defer r.mu.Unlock() + if r.started || r.stopped { + return + } + r.started = true + go func() { + defer close(r.done) + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + for { + if r.ctx.Err() != nil { + return + } + r.batch(r.ctx) + select { + case <-r.ctx.Done(): + return + case <-ticker.C: + } + } + }() +} +func (r *pgRawRuntime) Stop() { + r.mu.Lock() + r.stopped = true + r.cancel() + started := r.started + r.mu.Unlock() + if started { + <-r.done + } +} + +func prepareHostedPGServe(app config.Config, pg config.PGConfig, basePath string) (pgServeStartup, error) { + if err := pg.ValidateRawDerivation(app.RequireAuth); err != nil { + return pgServeStartup{}, err + } + applyClassifierConfig(app) + store, err := postgres.NewHostedStore(pg.URL, pg.Schema, pg.RawTenant, pg.AllowInsecure) + if err != nil { + return pgServeStartup{}, err + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + var runtime *pgRawRuntime + var closeUploads func() error + custody := &pgRawSyncCustody{dataDir: app.DataDir, tenant: pg.RawTenant, limits: rawsync.DefaultManifestLimits(), version: rawProcessingVersion()} + var once sync.Once + cleanup := func() { + once.Do(func() { + stop() + if runtime != nil { + runtime.Stop() + } + if closeUploads != nil { + if err := closeUploads(); err != nil { + log.Print("hosted raw upload cleanup failed") + } + } + if err := custody.Close(); err != nil { + log.Print("hosted raw custody cleanup failed") + } + if err := store.Close(); err != nil { + log.Print("hosted raw database cleanup failed") + } + }) + } + fail := func(err error) (pgServeStartup, error) { cleanup(); return pgServeStartup{}, err } + if err = applyRequiredCursorSecret(store, app); err != nil { + return fail(err) + } + store.SetCustomPricing(app.CustomModelPricing) + if err = postgres.CheckHostedRuntimeWritable(ctx, store.DB(), pg.Schema, app.ArchiveContent); err != nil { + return fail(err) + } + metadata, err := postgres.NewHostedRawIngestStore(store.DB(), pg.RawTenant, custody.version) + if err != nil { + return fail(err) + } + custody.metadata = metadata + authStore, err := postgres.NewTenantRawDeviceAuthStore(store.DB(), pg.RawTenant) + if err != nil { + return fail(err) + } + auth, err := rawsync.NewDeviceAuthService(authStore, pgRawSyncTokenTTL) + if err != nil { + return fail(err) + } + if pg.RawDerivation { + poll, attempt, maxAttempts := pg.RawWorkerBounds() + isolated, err := rawderive.NewSubprocessParser(time.Duration(attempt) * time.Second) + if err != nil { + return fail(err) + } + if err = isolated.Preflight(ctx); err != nil { + return fail(err) + } + retry := rawderive.RetryPolicy{Base: time.Second, Maximum: time.Minute, MaxAttempts: maxAttempts} + sink, err := postgres.NewRawProjectionStore(store.DB(), hostedRawProjectionOptions(app, pg.RawTenant, retry)) + if err != nil { + return fail(err) + } + worker, err := rawderive.NewWorker(rawderive.WorkerConfig{ + Queue: metadata, Manifests: rawderive.ManifestLoader{Store: custody, Limits: custody.limits}, + Materializer: rawderive.Materializer{Store: custody, BaseDir: os.TempDir(), MaxTotalBytes: 512 << 20}, Parser: isolated, Projection: sink, + Owner: fmt.Sprintf("hosted-%d", os.Getpid()), BatchSize: 1, LeaseDuration: time.Minute, HeartbeatInterval: 10 * time.Second, AttemptTimeout: time.Duration(attempt) * time.Second, + RetryBase: retry.Base, RetryMax: retry.Maximum, MaxAttempts: retry.MaxAttempts, + }) + if err != nil { + return fail(err) + } + runtime = newPGRawRuntime(ctx, time.Duration(poll)*time.Second, func(ctx context.Context) { + result, err := worker.RunBatch(ctx) + if err != nil && ctx.Err() == nil { + log.Printf("hosted raw worker: claimed=%d succeeded=%d retried=%d failed=%d; batch failed", result.Claimed, result.Succeeded, result.Retried, result.Failed) + } + if ctx.Err() != nil { + return + } + maintenance, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err = sink.SettlePendingSignals(maintenance, 64); err != nil && ctx.Err() == nil { + log.Print("hosted raw pending-signal maintenance failed") + } + }) + } + rtOpts := serveRuntimeOptions{Mode: "pg-serve", RequestedPort: app.Port} + app, err = prepareServeRuntimeConfig(app, rtOpts) + if err != nil { + return fail(err) + } + uploadOption, closeUploadStore, err := preparePGRawSyncUploads(app.DataDir, store.DB(), custody) + if err != nil { + return fail(err) + } + closeUploads = closeUploadStore + opts := []server.Option{server.WithVersion(server.VersionInfo{Version: version, Commit: commit, BuildDate: buildDate, ReadOnly: true}), server.WithDataDir(app.DataDir), server.WithBaseContext(ctx), server.WithRawSyncServices(auth, custody), server.WithRawSyncTenant(pg.RawTenant), uploadOption} + if basePath != "" { + opts = append(opts, server.WithBasePath(basePath)) + } + startup := pgServeStartup{cfg: app, ctx: ctx, rtOpts: rtOpts, srv: server.New(app, store, nil, opts...), cleanup: cleanup} + if runtime != nil { + startup.startWorker = runtime.Start + } + return startup, nil +} + +func newPGHostedProvisionCommand() *cobra.Command { + return &cobra.Command{Use: "hosted-provision [target]", Short: "Explicitly provision one hosted tenant schema using the configured owner connection", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.LoadMinimal() + if err != nil { + return err + } + target := "" + if len(args) > 0 { + target = args[0] + } + pg, err := cfg.ResolvePGTarget(target) + if err != nil { + return err + } + if pg.RawTenant == "" { + return errors.New("hosted-provision requires raw_tenant in the selected PG target") + } + applyClassifierConfig(cfg) + database, err := postgres.Open(pg.URL, pg.Schema, pg.AllowInsecure) + if err != nil { + return errors.New("opening hosted owner connection failed") + } + defer database.Close() + if err = postgres.EnsureHostedTenant(cmd.Context(), database, pg.Schema, pg.RawTenant); err != nil { + return fmt.Errorf("hosted provisioning failed: %w", err) + } + fmt.Fprintln(cmd.OutOrStdout(), "Hosted tenant schema provisioned. Use a restricted tenant runtime connection to serve.") + return nil + }} +} +func newPGRawReparseCommand() *cobra.Command { + var runID string + var batch int + cmd := &cobra.Command{Use: "raw-reparse [target]", Short: "Schedule one bounded, resumable batch of current raw heads for this parser version", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if runID == "" { + return errors.New("raw-reparse requires --run-id for its durable checkpoint") + } + cfg, err := config.LoadMinimal() + if err != nil { + return err + } + target := "" + if len(args) > 0 { + target = args[0] + } + pg, err := cfg.ResolvePGTarget(target) + if err != nil { + return err + } + if err = pg.ValidateRawDerivation(cfg.RequireAuth); err != nil { + return err + } + if pg.RawTenant == "" { + return errors.New("raw-reparse requires raw_tenant") + } + store, err := postgres.NewHostedStore(pg.URL, pg.Schema, pg.RawTenant, pg.AllowInsecure) + if err != nil { + return err + } + defer store.Close() + sink, err := postgres.NewRawProjectionStore(store.DB(), postgres.RawProjectionOptions{Tenant: pg.RawTenant}) + if err != nil { + return err + } + result, err := sink.ScheduleCurrentHeads(cmd.Context(), runID, rawProcessingVersion(), batch) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Selected %d current heads; complete=%t\n", result.Selected, result.Done) + return nil + }} + cmd.Flags().StringVar(&runID, "run-id", "", "Durable rollout checkpoint ID; reuse to resume") + cmd.Flags().IntVar(&batch, "batch-size", 64, "Maximum current heads in this invocation (1-256)") + return cmd +} + +func hostedRawProjectionOptions(app config.Config, tenant string, retry rawderive.RetryPolicy) postgres.RawProjectionOptions { + blocked := make(map[string]bool, len(app.ResultContentBlockedCategories)) + for _, category := range app.ResultContentBlockedCategories { + category = strings.TrimSpace(category) + if category != "" { + blocked[strings.ToUpper(category[:1])+strings.ToLower(category[1:])] = true + } + } + return postgres.RawProjectionOptions{Tenant: tenant, RetryPolicy: retry, Content: ingest.ContentOptions{ArchiveContent: app.ArchiveContent, ToolResultImages: app.ToolResultImages, BlockedResultCategories: blocked}} +} diff --git a/cmd/agentsview/pg_raw_derive_pgtest_test.go b/cmd/agentsview/pg_raw_derive_pgtest_test.go new file mode 100644 index 0000000000..093facabe0 --- /dev/null +++ b/cmd/agentsview/pg_raw_derive_pgtest_test.go @@ -0,0 +1,226 @@ +//go:build pgtest + +package main + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/postgres" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +func hostedRuntimeConfig(t *testing.T) (config.Config, *sql.DB) { + t.Helper() + dsn := os.Getenv("TEST_PG_URL") + if dsn == "" { + t.Skip("TEST_PG_URL not configured") + } + nonce := make([]byte, 16) + _, err := rand.Read(nonce) + require.NoError(t, err) + suffix := hex.EncodeToString(nonce) + schema := "runtime_test_" + suffix + role := schema + "_role" + admin, err := postgres.Open(dsn, schema, false) + require.NoError(t, err) + t.Cleanup(func() { + _, err := admin.ExecContext(context.Background(), `DROP SCHEMA "`+schema+`" CASCADE;DROP ROLE "`+role+`"`) + assert.NoError(t, err) + assert.NoError(t, admin.Close()) + }) + require.NoError(t, postgres.EnsureHostedTenant(t.Context(), admin, schema, "tenant-runtime")) + password := make([]byte, 32) + _, err = rand.Read(password) + require.NoError(t, err) + _, err = admin.Exec(`CREATE ROLE "` + role + `" LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD '` + hex.EncodeToString(password) + `';GRANT USAGE ON SCHEMA "` + schema + `" TO "` + role + `";GRANT SELECT,INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA "` + schema + `" TO "` + role + `";GRANT USAGE ON ALL SEQUENCES IN SCHEMA "` + schema + `" TO "` + role + `"`) + require.NoError(t, err) + u, err := url.Parse(dsn) + require.NoError(t, err) + u.User = url.UserPassword(role, hex.EncodeToString(password)) + cfg := config.Config{DataDir: t.TempDir(), Host: "127.0.0.1", Port: 0, RequireAuth: true, AuthToken: "synthetic-test-token", CursorSecret: "c3ludGhldGljLWN1cnNvci1zZWNyZXQ=", NoBrowser: true, PG: config.PGConfig{URL: u.String(), Schema: schema, RawTenant: "tenant-runtime"}} + return cfg, admin +} + +func TestHostedRuntimePreparationOffRetainsPublicReads(t *testing.T) { + cfg, _ := hostedRuntimeConfig(t) + // An occupied custody path makes any eager repository open fail decisively. + require.NoError(t, os.WriteFile(filepath.Join(cfg.DataDir, pgRawSyncDataDirectory), []byte("occupied"), 0600)) + startup, err := preparePGServeImpl(cfg, "") + require.NoError(t, err) + require.Nil(t, startup.startWorker) + startup.cleanup() + startup.cleanup() + store, closeStore, err := openPGReadStore(cfg, cfg.PG) + require.NoError(t, err) + defer closeStore() + _, ok := store.(*postgres.HostedStore) + require.True(t, ok) + cfg.PG.RawTenant = "" + _, err = preparePGServeImpl(cfg, "") + require.ErrorContains(t, err, "raw_tenant") +} +func TestHostedRuntimePreparationSandboxAndIdle(t *testing.T) { + cfg, _ := hostedRuntimeConfig(t) + cfg.PG.RawDerivation = true + p, err := rawderive.NewSubprocessParser(5 * time.Second) + require.NoError(t, err) + supported := p.Preflight(t.Context()) == nil + if !supported && os.Getenv("RAW_SANDBOX_REQUIRED") == "1" { + t.Fatal("mandatory sandbox unavailable") + } + require.NoError(t, os.WriteFile(filepath.Join(cfg.DataDir, pgRawSyncDataDirectory), []byte("occupied"), 0600)) + startup, err := preparePGServeImpl(cfg, "") + if !supported { + require.ErrorIs(t, err, rawderive.ErrSandboxUnavailable) + return + } + require.NoError(t, err) + defer startup.cleanup() + require.NotNil(t, startup.startWorker) + startup.startWorker() + // This cancellation joins the real empty queue/maintenance pass. Occupied + // custody proves startup and empty work have no repository dependency. + startup.cleanup() +} + +// This gate executes the actual hosted startup worker and parser child. It is +// mandatory on the Linux isolation runner; the local restricted host skips it. +func TestHostedRuntimeDerivesAcceptedSource(t *testing.T) { + p, err := rawderive.NewSubprocessParser(5 * time.Second) + require.NoError(t, err) + if err = p.Preflight(t.Context()); err != nil { + if os.Getenv("RAW_SANDBOX_REQUIRED") == "1" { + t.Fatal(err) + } + t.Skip("kernel isolation unavailable") + } + cfg, _ := hostedRuntimeConfig(t) + cfg.PG.RawDerivation = true + cfg.PG.RawPollSeconds = 1 + startup, err := preparePGServeImpl(cfg, "") + require.NoError(t, err) + defer startup.cleanup() + database, err := postgres.OpenHosted(cfg.PG.URL, cfg.PG.Schema, cfg.PG.RawTenant, false) + require.NoError(t, err) + defer database.Close() + authStore, err := postgres.NewTenantRawDeviceAuthStore(database, cfg.PG.RawTenant) + require.NoError(t, err) + auth, err := rawsync.NewDeviceAuthService(authStore, time.Minute) + require.NoError(t, err) + enrolled, err := auth.EnrollDevice(t.Context(), cfg.PG.RawTenant, "synthetic device") + require.NoError(t, err) + request := func(method, path, token, kind string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, "http://127.0.0.1"+path, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", kind) + req.Header.Set("X-AgentsView-Device-ID", enrolled.Identity.DeviceID) + req.Header.Set("Upload-Offset", "0") + rec := httptest.NewRecorder() + startup.srv.Handler().ServeHTTP(rec, req) + return rec + } + tokenResponse := request("POST", "/api/v1/raw-sync/tokens", enrolled.Credential, "application/json", []byte(`{"scopes":["upload","commit"]}`)) + require.Equal(t, 200, tokenResponse.Code) + var token struct{ Token string } + require.NoError(t, json.Unmarshal(tokenResponse.Body.Bytes(), &token)) + body := hostedRuntimeClaudeFixture() + digest := sha256.Sum256(body) + ref := rawsync.ObjectRef{SHA256: hex.EncodeToString(digest[:]), Length: int64(len(body))} + encoded, err := json.Marshal(map[string]any{"provider": "claude", "object": ref}) + require.NoError(t, err) + uploadResponse := request("POST", "/api/v1/raw-sync/uploads", token.Token, "application/json", encoded) + require.Contains(t, []int{200, 201}, uploadResponse.Code) + var upload struct { + UploadID string `json:"upload_id"` + } + require.NoError(t, json.Unmarshal(uploadResponse.Body.Bytes(), &upload)) + require.NotEmpty(t, upload.UploadID) + appended := request("PATCH", "/api/v1/raw-sync/uploads/"+upload.UploadID, token.Token, "application/octet-stream", body) + require.Equal(t, 200, appended.Code) + manifest := rawsync.Manifest{SchemaVersion: rawsync.ManifestSchemaVersion, Provider: parser.AgentClaude, ConfiguredRootID: "root", SourceKey: "/canonical/project/runtime-session.jsonl", CaptureID: "runtime-capture", CapturedAt: time.Now().UTC(), Kind: rawsync.ManifestSnapshot, Entries: []rawsync.Entry{{Path: "project/runtime-session.jsonl", Type: "file", Length: ref.Length, Objects: []rawsync.ObjectRef{ref}}}} + encoded, err = json.Marshal(manifest) + require.NoError(t, err) + accepted := request("POST", "/api/v1/raw-sync/manifests", token.Token, "application/json", encoded) + require.Equal(t, 200, accepted.Code, accepted.Body.String()) + var attempts, generation int + require.NoError(t, database.QueryRow(`SELECT attempt_count,projection_generation FROM raw_ingest_jobs`).Scan(&attempts, &generation)) + assert.Zero(t, attempts) + assert.Equal(t, 1, generation) + ctx, cancel := context.WithCancel(startup.ctx) + startup.ctx = ctx + defer cancel() + finished := make(chan error, 1) + go func() { finished <- runPreparedPGServe(startup) }() + defer func() { + cancel() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Error("hosted runtime did not join") + } + }() + require.Eventually(t, func() bool { + var state string + err := database.QueryRow(`SELECT state FROM raw_ingest_jobs`).Scan(&state) + return err == nil && state == "complete" + }, 20*time.Second, 50*time.Millisecond) + store, err := postgres.NewHostedStore(cfg.PG.URL, cfg.PG.Schema, cfg.PG.RawTenant, false) + require.NoError(t, err) + defer store.Close() + messages, err := store.GetMessages(t.Context(), "runtime-session", 0, 10, true) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "hello runtime", messages[0].Content) + assert.Equal(t, "hello viewer", messages[1].Content) +} + +func TestHostedRuntimeRejectsMissingPublicationPrivilegesBeforeReadiness(t *testing.T) { + cfg, admin := hostedRuntimeConfig(t) + endpoint, err := url.Parse(cfg.PG.URL) + require.NoError(t, err) + role := endpoint.User.Username() + control, err := preparePGServeImpl(cfg, "") + require.NoError(t, err) + control.cleanup() + cfg.PG.RawDerivation = true + for _, tc := range []struct{ object, privilege, kind string }{ + {"session_sources", "DELETE", "TABLE"}, {"raw_session_links", "DELETE", "TABLE"}, {"tool_result_events", "INSERT", "TABLE"}, {"usage_events", "INSERT", "TABLE"}, {"secret_findings", "INSERT", "TABLE"}, {"starred_sessions", "INSERT", "TABLE"}, {"pinned_messages", "UPDATE", "TABLE"}, + {"tool_calls_id_seq", "USAGE", "SEQUENCE"}, {"tool_result_events_id_seq", "USAGE", "SEQUENCE"}, {"usage_events_id_seq", "USAGE", "SEQUENCE"}, {"pinned_messages_id_seq", "USAGE", "SEQUENCE"}, + } { + t.Run(tc.object+"_"+tc.privilege, func(t *testing.T) { + _, err := admin.Exec(`REVOKE ` + tc.privilege + ` ON ` + tc.kind + ` "` + tc.object + `" FROM "` + role + `"`) + require.NoError(t, err) + defer func() { + _, err := admin.Exec(`GRANT ` + tc.privilege + ` ON ` + tc.kind + ` "` + tc.object + `" TO "` + role + `"`) + require.NoError(t, err) + }() + startup, err := preparePGServeImpl(cfg, "") + if err == nil { + startup.cleanup() + } + require.ErrorContains(t, err, "privileges") + require.Nil(t, startup.startWorker) + var attempts int + require.NoError(t, admin.QueryRow(`SELECT COALESCE(sum(attempt_count),0) FROM raw_ingest_jobs`).Scan(&attempts)) + assert.Zero(t, attempts) + }) + } +} diff --git a/cmd/agentsview/pg_raw_derive_test.go b/cmd/agentsview/pg_raw_derive_test.go new file mode 100644 index 0000000000..edfae4c851 --- /dev/null +++ b/cmd/agentsview/pg_raw_derive_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/server" +) + +func TestPGRawRuntimeChild(t *testing.T) { + if os.Getenv("RAW_RUNTIME_CHILD") != "1" { + return + } + for { + time.Sleep(time.Hour) + } +} + +func TestPGRawRuntimeReadinessCancellationJoinsChild(t *testing.T) { + var calls atomic.Int32 + entered := make(chan *exec.Cmd, 1) + joined := make(chan struct{}) + r := newPGRawRuntime(t.Context(), time.Hour, func(ctx context.Context) { + calls.Add(1) + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPGRawRuntimeChild$") + cmd.Env = []string{"RAW_RUNTIME_CHILD=1"} + if err := cmd.Start(); err != nil { + entered <- nil + return + } + entered <- cmd + _ = cmd.Wait() + close(joined) + }) + assert.Zero(t, calls.Load(), "preparation must not start claims") + r.Start() + r.Start() + var child *exec.Cmd + select { + case child = <-entered: + case <-time.After(3 * time.Second): + t.Fatal("worker did not start") + } + require.NotNil(t, child) + r.Stop() + select { + case <-joined: + default: + t.Fatal("Stop returned before child Wait") + } + require.NotNil(t, child.ProcessState) + r.Start() + r.Stop() + assert.EqualValues(t, 1, calls.Load()) +} +func TestPGRawRuntimeClosedBeforeReadinessNeverStarts(t *testing.T) { + var calls atomic.Int32 + r := newPGRawRuntime(t.Context(), time.Second, func(context.Context) { calls.Add(1) }) + r.Stop() + r.Start() + r.Stop() + assert.Zero(t, calls.Load()) +} + +func TestMain(m *testing.M) { + if handled, code := rawderive.RunParserChild(os.Args[1:]); handled { + os.Exit(code) + } + os.Exit(m.Run()) +} + +func TestPGRawRuntimeReadinessFailureCleansBeforeReturn(t *testing.T) { + var cleaned, started atomic.Bool + cfg := config.Config{DataDir: t.TempDir(), Host: "invalid host", Port: 12345, NoBrowser: true} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + startup := pgServeStartup{cfg: cfg, ctx: ctx, srv: server.New(cfg, nil, nil), rtOpts: serveRuntimeOptions{Mode: "pg-serve", RequestedPort: cfg.Port}, cleanup: func() { cleaned.Store(true) }, startWorker: func() { started.Store(true) }} + require.Error(t, runPreparedPGServe(startup)) + assert.True(t, cleaned.Load()) + assert.False(t, started.Load()) +} + +func TestHostedProjectionConfiguredContentPolicies(t *testing.T) { + parsed := parser.ParseResult{Session: parser.ParsedSession{ID: "policy-session", Agent: parser.AgentClaude}, Messages: []parser.ParsedMessage{ + {Ordinal: 0, Role: parser.RoleAssistant, Content: "checking", ToolCalls: []parser.ParsedToolCall{{ToolUseID: "read", ToolName: "Read", Category: "Read", InputJSON: `{"path":"file"}`}, {ToolUseID: "bash", ToolName: "Bash", Category: "Bash", InputJSON: `{"command":"true"}`}}}, + {Ordinal: 1, Role: parser.RoleUser, ToolResults: []parser.ParsedToolResult{{ToolUseID: "read", ContentRaw: `"private result"`, ContentLength: 14}, {ToolUseID: "bash", ContentRaw: `[{"type":"text","text":"before"},{"type":"input_image","image_url":"data:image/png;base64,AAEC"}]`}}}, + }} + imageContent, err := json.Marshal(`[{"type":"text","text":"before"},{"type":"input_image","image_url":"data:image/png;base64,AAEC"}]`) + require.NoError(t, err) + parsed.Messages[1].ToolResults[1].ContentRaw = string(imageContent) + for _, mode := range []string{"full", "blocked", "images", "transcripts"} { + t.Run(mode, func(t *testing.T) { + app := config.Config{} + switch mode { + case "blocked": + app.ResultContentBlockedCategories = []string{" rEaD ", " "} + case "images": + app.ToolResultImages = config.ToolResultImagesDrop + case "transcripts": + app.ArchiveContent = config.ArchiveContentTranscripts + } + options := hostedRawProjectionOptions(app, "tenant", rawderive.RetryPolicy{}).Content + candidate, err := ingest.PrepareCandidate(t.Context(), parsed, options) + require.NoError(t, err) + prepared, err := ingest.Finalize(t.Context(), candidate, options) + require.NoError(t, err) + require.Len(t, prepared.Messages, 1) + require.Len(t, prepared.Messages[0].ToolCalls, 2) + read, bash := prepared.Messages[0].ToolCalls[0], prepared.Messages[0].ToolCalls[1] + if mode == "blocked" || mode == "transcripts" { + assert.Empty(t, read.ResultContent) + } else { + assert.Equal(t, "private result", read.ResultContent) + } + if mode == "images" || mode == "transcripts" { + assert.NotContains(t, bash.ResultContent, "base64,AAEC") + } else { + assert.Contains(t, bash.ResultContent, "base64,AAEC") + } + if mode == "transcripts" { + assert.Empty(t, bash.InputJSON) + } else { + assert.Equal(t, `{"command":"true"}`, bash.InputJSON) + } + }) + } +} + +func hostedRuntimeClaudeFixture() []byte { + return []byte(`{"type":"user","timestamp":"2026-08-13T12:00:00Z","uuid":"u1","sessionId":"runtime-session","message":{"content":"hello runtime"},"cwd":"/work/project"}` + "\n" + `{"type":"assistant","timestamp":"2026-08-13T12:00:01Z","uuid":"a1","parentUuid":"u1","sessionId":"runtime-session","message":{"content":"hello viewer"}}` + "\n") +} + +func TestHostedRuntimeClaudeFixtureHasTwoPreparedMessages(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "project", "runtime-session.jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, hostedRuntimeClaudeFixture(), 0400)) + factory, ok := parser.ProviderFactoryByType(parser.AgentClaude) + require.True(t, ok) + provider := factory.NewProvider(parser.ProviderConfig{Roots: []string{root}, Machine: "hosted"}) + outcome, err := provider.Parse(parser.WithoutFilesystemProjectDiscovery(t.Context()), parser.ParseRequest{Source: parser.SourceRef{Provider: parser.AgentClaude, DisplayPath: path, Key: path, ProjectHint: "project"}}) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + candidate, err := ingest.PrepareCandidate(t.Context(), outcome.Results[0].Result, ingest.ContentOptions{}) + require.NoError(t, err) + prepared, err := ingest.Finalize(t.Context(), candidate, ingest.ContentOptions{}) + require.NoError(t, err) + require.Len(t, prepared.Messages, 2) + assert.Equal(t, "runtime-session", prepared.Session.ID) + assert.Equal(t, "user", prepared.Messages[0].Role) + assert.Equal(t, "hello runtime", prepared.Messages[0].Content) + assert.Equal(t, "assistant", prepared.Messages[1].Role) + assert.Equal(t, "hello viewer", prepared.Messages[1].Content) +} diff --git a/cmd/agentsview/pg_raw_parity_pgtest_test.go b/cmd/agentsview/pg_raw_parity_pgtest_test.go new file mode 100644 index 0000000000..c06dbf23e4 --- /dev/null +++ b/cmd/agentsview/pg_raw_parity_pgtest_test.go @@ -0,0 +1,436 @@ +//go:build pgtest + +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/postgres" + "go.kenn.io/agentsview/internal/rawcapture" + "go.kenn.io/agentsview/internal/rawcheckpoint" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" + "go.kenn.io/agentsview/internal/rawtest" +) + +type hostedCaptureClient struct { + checkpoint *rawcheckpoint.Store + provider parser.Provider + device, token string + startup *pgServeStartup + last rawsync.Manifest +} + +func requireHostedSandbox(t *testing.T) { + t.Helper() + p, err := rawderive.NewSubprocessParser(5 * time.Second) + require.NoError(t, err) + if err = p.Preflight(t.Context()); err != nil { + if os.Getenv("RAW_SANDBOX_REQUIRED") == "1" { + t.Fatal(err) + } + t.Skip("kernel isolation unavailable") + } +} +func startParityRuntime(t *testing.T, policy config.ArchiveContent) (*pgServeStartup, *postgres.HostedStore, *sql.DB) { + t.Helper() + requireHostedSandbox(t) + cfg, admin := hostedRuntimeConfig(t) + cfg.PG.RawDerivation = true + cfg.PG.RawPollSeconds = 1 + cfg.PG.RawMaxAttempts = 2 + cfg.ArchiveContent = policy + // An occupied archive path makes an accidental hosted archive open fail. + cfg.DBPath = filepath.Join(cfg.DataDir, "sessions.db") + require.NoError(t, os.Mkdir(cfg.DBPath, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(cfg.DBPath, "sentinel"), []byte("not an archive"), 0600)) + startup, err := preparePGServeImpl(cfg, "") + require.NoError(t, err) + ctx, cancel := context.WithCancel(startup.ctx) + startup.ctx = ctx + finished := make(chan error, 1) + go func() { finished <- runPreparedPGServe(startup) }() + t.Cleanup(func() { + cancel() + select { + case err := <-finished: + assert.NoError(t, err) + case <-time.After(10 * time.Second): + t.Error("runtime failed to join") + } + startup.cleanup() + body, err := os.ReadFile(filepath.Join(cfg.DBPath, "sentinel")) + assert.NoError(t, err) + assert.Equal(t, "not an archive", string(body)) + }) + store, err := postgres.NewHostedStore(cfg.PG.URL, cfg.PG.Schema, cfg.PG.RawTenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return &startup, store, admin +} +func newHostedCaptureClient(t *testing.T, startup *pgServeStartup, store *postgres.HostedStore, agent parser.AgentType, root string) *hostedCaptureClient { + t.Helper() + authStore, err := postgres.NewTenantRawDeviceAuthStore(store.DB(), "tenant-runtime") + require.NoError(t, err) + auth, err := rawsync.NewDeviceAuthService(authStore, time.Minute) + require.NoError(t, err) + enrolled, err := auth.EnrollDevice(t.Context(), "tenant-runtime", "synthetic capture device") + require.NoError(t, err) + provider, ok := parser.NewProvider(agent, parser.ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + dir := t.TempDir() + cp, err := rawcheckpoint.OpenWithOptions(t.Context(), filepath.Join(dir, "checkpoint.db"), rawcheckpoint.Options{SpoolDir: filepath.Join(dir, "spool"), MaxOutboxBytes: 8 << 20}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, cp.Close()) }) + require.NoError(t, cp.SetDevice(t.Context(), enrolled.Identity.DeviceID)) + c := &hostedCaptureClient{checkpoint: cp, provider: provider, device: enrolled.Identity.DeviceID, startup: startup} + reply := c.request("POST", "/api/v1/raw-sync/tokens", enrolled.Credential, "application/json", []byte(`{"scopes":["upload","commit"]}`)) + require.Equal(t, 200, reply.Code, reply.Body.String()) + var token struct{ Token string } + require.NoError(t, json.Unmarshal(reply.Body.Bytes(), &token)) + c.token = token.Token + return c +} +func (c *hostedCaptureClient) request(method, path, token, kind string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, "http://127.0.0.1"+path, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", kind) + req.Header.Set("X-AgentsView-Device-ID", c.device) + req.Header.Set("Upload-Offset", "0") + rec := httptest.NewRecorder() + c.startup.srv.Handler().ServeHTTP(rec, req) + return rec +} +func (c *hostedCaptureClient) commit(t *testing.T, m rawsync.Manifest) rawsync.CommitResult { + t.Helper() + body, err := json.Marshal(m) + require.NoError(t, err) + rec := c.request("POST", "/api/v1/raw-sync/manifests", c.token, "application/json", body) + require.Equal(t, 200, rec.Code, rec.Body.String()) + var result struct { + ManifestID string `json:"manifest_id"` + Receipt string `json:"receipt"` + Generation int64 `json:"generation"` + Created bool `json:"created"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &result)) + return rawsync.CommitResult{ManifestID: result.ManifestID, Receipt: result.Receipt, Generation: result.Generation, Created: result.Created} +} +func (c *hostedCaptureClient) flush(t *testing.T) rawsync.CommitResult { + t.Helper() + m, found, err := c.checkpoint.FinalizeNextManifest(t.Context(), c.device) + require.NoError(t, err) + require.True(t, found) + for _, entry := range m.Entries { + for _, ref := range entry.Objects { + body, err := json.Marshal(map[string]any{"provider": m.Provider, "object": ref}) + require.NoError(t, err) + rec := c.request("POST", "/api/v1/raw-sync/uploads", c.token, "application/json", body) + require.Contains(t, []int{200, 201}, rec.Code, rec.Body.String()) + var upload struct { + UploadID string `json:"upload_id"` + Complete bool `json:"complete"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &upload)) + if !upload.Complete { + require.NotEmpty(t, upload.UploadID) + payload, err := os.ReadFile(c.checkpoint.ObjectPath(ref)) + require.NoError(t, err) + rec = c.request("PATCH", "/api/v1/raw-sync/uploads/"+upload.UploadID, c.token, "application/octet-stream", payload) + require.Equal(t, 200, rec.Code, rec.Body.String()) + } + } + } + result := c.commit(t, m) + require.NoError(t, c.checkpoint.BindFinalizedCommit(t.Context(), c.device, m.CaptureID, result)) + _, err = c.checkpoint.AcknowledgeGeneration(t.Context(), c.device, m.CaptureID, result) + require.NoError(t, err) + c.last = m + return result +} +func (c *hostedCaptureClient) capture(t *testing.T) rawsync.CommitResult { + t.Helper() + sources, err := parser.DiscoverRawCaptureSources(t.Context(), c.provider) + require.NoError(t, err) + require.True(t, sources.Complete) + require.Len(t, sources.Sources, 1) + result, err := rawcapture.New(c.checkpoint).Capture(t.Context(), c.provider, sources.Sources[0]) + require.NoError(t, err) + require.Equal(t, rawcapture.StatusCaptured, result.Status) + return c.flush(t) +} +func waitCaptureJob(t *testing.T, pg *sql.DB, manifest, state string, attempts int) { + t.Helper() + require.Eventually(t, func() bool { + var got string + var count int + err := pg.QueryRow(`SELECT state,attempt_count FROM raw_ingest_jobs WHERE manifest_id=$1`, manifest).Scan(&got, &count) + return err == nil && got == state && count == attempts + }, 30*time.Second, 50*time.Millisecond) +} + +// Losing any captured companion, parser wire field, configured policy or +// publication child row must diverge from the independently synced SQLite rows. +func TestHostedRuntimeCapturedParity(t *testing.T) { + for _, tc := range []struct { + name string + agent parser.AgentType + policy config.ArchiveContent + }{ + {"claude", parser.AgentClaude, config.ArchiveContentFull}, {"claude_transcript", parser.AgentClaude, config.ArchiveContentTranscripts}, {"zcode", parser.AgentZCode, config.ArchiveContentFull}, {"codex_tools", parser.AgentCodex, config.ArchiveContentFull}, + } { + t.Run(tc.name, func(t *testing.T) { + startup, store, admin := startParityRuntime(t, tc.policy) + root := t.TempDir() + ids := []string{rawtest.ClaudeID} + var path string + if tc.agent == parser.AgentClaude { + path = rawtest.Claude(t, root) + } else if tc.agent == parser.AgentZCode { + rawtest.ZCode(t, root) + ids = []string{rawtest.ZCodeID, rawtest.ZCodeBillableID} + } else { + rawtest.CodexTools(t, root) + ids = []string{"codex:" + rawtest.CodexToolsID} + } + oracle, engine := rawtest.Oracle(t, tc.agent, root, tc.policy, "") + require.Equal(t, len(ids), engine.SyncAll(t.Context(), nil).Synced) + client := newHostedCaptureClient(t, startup, store, tc.agent, root) + receipt := client.capture(t) + waitCaptureJob(t, admin, receipt.ManifestID, "complete", 1) + rawtest.EqualStored(t, t.Context(), oracle, store, ids...) + core, err := postgres.NewRawProjectionStore(store.DB(), postgres.RawProjectionOptions{Tenant: "tenant-runtime"}) + require.NoError(t, err) + for _, id := range ids { + resolved, err := core.Resolve(t.Context(), id) + require.NoError(t, err) + rawtest.EqualUsageEvents(t, t.Context(), oracle, store.DB(), id, resolved.SessionID) + } + if tc.agent == parser.AgentCodex { + return + } + if tc.agent == parser.AgentZCode { + messages, err := store.GetAllMessages(t.Context(), rawtest.ZCodeBillableID) + require.NoError(t, err) + assert.Empty(t, messages) + usage, err := store.GetSessionUsage(t.Context(), rawtest.ZCodeBillableID, true) + require.NoError(t, err) + require.NotNil(t, usage) + assert.Equal(t, 17, usage.TotalOutputTokens) + assert.True(t, usage.HasCost) + return + } + if tc.policy == config.ArchiveContentFull { + require.Greater(t, len(client.last.Entries), 1, "capture must include persisted tool output") + } + require.NoError(t, store.RenameSession(rawtest.ClaudeID, new("Curated title"))) + require.NoError(t, oracle.RenameSession(rawtest.ClaudeID, new("Curated title"))) + _, err = store.StarSession(rawtest.ClaudeID) + require.NoError(t, err) + _, err = store.PinMessage(rawtest.ClaudeID, 0, new("Review this prompt")) + require.NoError(t, err) + sources, err := parser.DiscoverRawCaptureSources(t.Context(), client.provider) + require.NoError(t, err) + require.Len(t, sources.Sources, 1) + unchanged, err := rawcapture.New(client.checkpoint).Capture(t.Context(), client.provider, sources.Sources[0]) + require.NoError(t, err) + assert.Equal(t, rawcapture.StatusUnchanged, unchanged.Status) + var revision int64 + require.NoError(t, admin.QueryRow(`SELECT corpus_revision FROM raw_corpus_state`).Scan(&revision)) + replay := client.commit(t, client.last) + assert.False(t, replay.Created) + var after int64 + require.NoError(t, admin.QueryRow(`SELECT corpus_revision FROM raw_corpus_state`).Scan(&after)) + assert.Equal(t, revision, after) + waitCaptureJob(t, admin, receipt.ManifestID, "complete", 1) + rawtest.AppendClaude(t, path) + require.Equal(t, 1, engine.SyncAllForceParse(t.Context(), nil).Synced) + appended := client.capture(t) + waitCaptureJob(t, admin, appended.ManifestID, "complete", 1) + rawtest.EqualStored(t, t.Context(), oracle, store, ids...) + pins, err := store.ListPinnedMessages(t.Context(), rawtest.ClaudeID, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Zero(t, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "Review this prompt", *pins[0].Note) + stars, err := store.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{rawtest.ClaudeID}, stars) + }) + } +} + +// Equal device content coalesces, divergent content becomes explicitly +// ambiguous, and a captured tombstone removes only that device's proof. +func TestHostedRuntimeCapturedConflictAndRemoval(t *testing.T) { + startup, store, admin := startParityRuntime(t, config.ArchiveContentFull) + rootA, rootB := t.TempDir(), t.TempDir() + rawtest.Claude(t, rootA) + pathB := rawtest.Claude(t, rootB) + a := newHostedCaptureClient(t, startup, store, parser.AgentClaude, rootA) + b := newHostedCaptureClient(t, startup, store, parser.AgentClaude, rootB) + first := a.capture(t) + waitCaptureJob(t, admin, first.ManifestID, "complete", 1) + second := b.capture(t) + waitCaptureJob(t, admin, second.ManifestID, "complete", 1) + session, err := store.GetSession(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + require.NotNil(t, session) + require.NoError(t, store.RenameSession(rawtest.ClaudeID, new("Shared curation"))) + rawtest.AppendClaude(t, pathB) + divergent := b.capture(t) + waitCaptureJob(t, admin, divergent.ManifestID, "complete", 1) + _, err = store.GetSession(t.Context(), rawtest.ClaudeID) + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + assert.Equal(t, "ambiguous", conflict.State) + require.Len(t, conflict.Variants, 2) + var contents []string + for _, id := range conflict.Variants { + messages, err := store.GetAllMessages(t.Context(), id) + require.NoError(t, err) + require.NotEmpty(t, messages) + contents = append(contents, messages[len(messages)-1].Content) + } + assert.ElementsMatch(t, []string{"The build failed.", "Recorded the failure."}, contents) + _, queued, err := b.checkpoint.QueueTombstone(t.Context(), rawcheckpoint.SourceIdentity{Provider: b.last.Provider, ConfiguredRootID: b.last.ConfiguredRootID, SourceKey: b.last.SourceKey}) + require.NoError(t, err) + require.True(t, queued) + removed := b.flush(t) + waitCaptureJob(t, admin, removed.ManifestID, "complete", 1) + session, err = store.GetSession(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + require.NotNil(t, session) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "Shared curation", *session.DisplayName) + messages, err := store.GetAllMessages(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + require.NotEmpty(t, messages) + assert.Equal(t, "The build failed.", messages[len(messages)-1].Content) + _, queued, err = a.checkpoint.QueueTombstone(t.Context(), rawcheckpoint.SourceIdentity{Provider: a.last.Provider, ConfiguredRootID: a.last.ConfiguredRootID, SourceKey: a.last.SourceKey}) + require.NoError(t, err) + require.True(t, queued) + removed = a.flush(t) + waitCaptureJob(t, admin, removed.ManifestID, "complete", 1) + session, err = store.GetSession(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + assert.Nil(t, session) +} + +// A missing fork parent is a real provider partial result: publish its visible +// messages, retry finitely, and recover when capture includes the parent. +func TestHostedRuntimeCapturedPartialRetryExhaustion(t *testing.T) { + startup, store, admin := startParityRuntime(t, config.ArchiveContentFull) + root := t.TempDir() + rawtest.CodexFork(t, root) + client := newHostedCaptureClient(t, startup, store, parser.AgentCodex, root) + partial := client.capture(t) + waitCaptureJob(t, admin, partial.ManifestID, "failed", 2) + id := "codex:" + rawtest.CodexChildID + messages, err := store.GetAllMessages(t.Context(), id) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "Inspect the fork.", messages[0].Content) + assert.Equal(t, "Fork inspected.", messages[1].Content) + require.NoError(t, store.RenameSession(id, new("Partial curation"))) + var revision int64 + require.NoError(t, admin.QueryRow(`SELECT corpus_revision FROM raw_corpus_state`).Scan(&revision)) + replay := client.commit(t, client.last) + assert.False(t, replay.Created) + core, err := postgres.NewRawProjectionStore(store.DB(), postgres.RawProjectionOptions{Tenant: "tenant-runtime"}) + require.NoError(t, err) + rollout, err := core.ScheduleCurrentHeads(t.Context(), "same-version-replay", rawProcessingVersion(), 64) + require.NoError(t, err) + assert.True(t, rollout.Done) + // Three worker polls after failure must not claim an exhausted generation. + require.Never(t, func() bool { + var attempts int + err := admin.QueryRow(`SELECT attempt_count FROM raw_ingest_jobs WHERE manifest_id=$1`, partial.ManifestID).Scan(&attempts) + return err != nil || attempts != 2 + }, 3200*time.Millisecond, 100*time.Millisecond) + var after int64 + require.NoError(t, admin.QueryRow(`SELECT corpus_revision FROM raw_corpus_state`).Scan(&after)) + assert.Equal(t, revision, after) + rawtest.CodexParent(t, root) + sources, err := parser.DiscoverRawCaptureSources(t.Context(), client.provider) + require.NoError(t, err) + var captured bool + for _, source := range sources.Sources { + if source.Key == rawtest.CodexChildID || strings.Contains(source.DisplayPath, rawtest.CodexChildID) { + result, err := rawcapture.New(client.checkpoint).Capture(t.Context(), client.provider, source) + require.NoError(t, err) + require.Equal(t, rawcapture.StatusCaptured, result.Status) + captured = true + } + } + require.True(t, captured) + recovered := client.flush(t) + require.Greater(t, len(client.last.Entries), 1, "the parent must be captured as a companion") + waitCaptureJob(t, admin, recovered.ManifestID, "complete", 1) + oracle, engine := rawtest.Oracle(t, parser.AgentCodex, root, "", "") + require.Positive(t, engine.SyncAll(t.Context(), nil).Synced) + require.NoError(t, oracle.RenameSession(id, new("Partial curation"))) + rawtest.EqualStored(t, t.Context(), oracle, store, id) +} + +func TestHostedRuntimeCapturedRelationships(t *testing.T) { + for _, parentFirst := range []bool{true, false} { + name := "child_first" + if parentFirst { + name = "parent_first" + } + t.Run(name, func(t *testing.T) { + startup, store, admin := startParityRuntime(t, config.ArchiveContentFull) + root := t.TempDir() + rawtest.Claude(t, root) + rawtest.ClaudeChild(t, root) + oracle, engine := rawtest.Oracle(t, parser.AgentClaude, root, "", "") + require.Equal(t, 2, engine.SyncAll(t.Context(), nil).Synced) + client := newHostedCaptureClient(t, startup, store, parser.AgentClaude, root) + sources, err := parser.DiscoverRawCaptureSources(t.Context(), client.provider) + require.NoError(t, err) + require.Len(t, sources.Sources, 2) + slices.SortFunc(sources.Sources, func(a, b parser.SourceRef) int { + aChild := strings.Contains(a.DisplayPath, rawtest.ClaudeChildID) + bChild := strings.Contains(b.DisplayPath, rawtest.ClaudeChildID) + if aChild == bChild { + return 0 + } + if aChild == parentFirst { + return 1 + } + return -1 + }) + require.Equal(t, !parentFirst, strings.Contains(sources.Sources[0].DisplayPath, rawtest.ClaudeChildID)) + for _, source := range sources.Sources { + result, err := rawcapture.New(client.checkpoint).Capture(t.Context(), client.provider, source) + require.NoError(t, err) + require.Equal(t, rawcapture.StatusCaptured, result.Status) + receipt := client.flush(t) + waitCaptureJob(t, admin, receipt.ManifestID, "complete", 1) + } + rawtest.EqualStored(t, t.Context(), oracle, store, rawtest.ClaudeID, rawtest.ClaudeChildID) + children, err := store.GetChildSessions(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + require.Len(t, children, 1) + assert.Equal(t, rawtest.ClaudeChildID, children[0].ID) + + }) + } +} diff --git a/cmd/agentsview/pg_raw_sync.go b/cmd/agentsview/pg_raw_sync.go index 291873d03d..bc8146f156 100644 --- a/cmd/agentsview/pg_raw_sync.go +++ b/cmd/agentsview/pg_raw_sync.go @@ -78,6 +78,8 @@ type pgRawSyncCustody struct { mu sync.Mutex dataDir string + tenant string + objects rawsync.ObjectStore metadata rawsync.MetadataStore limits rawsync.ManifestLimits version string @@ -93,6 +95,9 @@ func (c *pgRawSyncCustody) MissingObjects( provider parser.AgentType, objects []rawsync.ObjectRef, ) ([]rawsync.ObjectRef, error) { + if c.tenant != "" && identity.TenantID != c.tenant { + return nil, rawsync.ErrUnauthorized + } service, err := c.openService(ctx) if err != nil { return nil, err @@ -107,6 +112,9 @@ func (c *pgRawSyncCustody) FinalizeObject( object rawsync.ObjectRef, body io.Reader, ) (rawsync.PutResult, error) { + if c.tenant != "" && identity.TenantID != c.tenant { + return rawsync.PutResult{}, rawsync.ErrUnauthorized + } service, err := c.openService(ctx) if err != nil { return rawsync.PutResult{}, err @@ -119,6 +127,9 @@ func (c *pgRawSyncCustody) CommitManifest( identity rawsync.AuthIdentity, manifest rawsync.Manifest, ) (rawsync.CommitResult, error) { + if c.tenant != "" && identity.TenantID != c.tenant { + return rawsync.CommitResult{}, rawsync.ErrUnauthorized + } service, err := c.openService(ctx) if err != nil { return rawsync.CommitResult{}, err @@ -153,6 +164,7 @@ func (c *pgRawSyncCustody) openService(ctx context.Context) (*rawsync.Service, e if err != nil { return fail(fmt.Errorf("preparing raw sync custody service: %w", err)) } + c.objects = objects c.repository = repository c.service = service return service, nil @@ -197,3 +209,23 @@ func combinePGRawSyncOptions(options ...server.Option) server.Option { } } } + +// The worker shares custody's single lazily opened exclusive repository. +func (c *pgRawSyncCustody) OpenManifest(ctx context.Context, identity rawsync.AuthIdentity, id string) (rawsync.ObjectInfo, rawsync.VerifiedObjectReader, error) { + if c.tenant != "" && identity.TenantID != c.tenant { + return rawsync.ObjectInfo{}, nil, rawsync.ErrUnauthorized + } + if _, err := c.openService(ctx); err != nil { + return rawsync.ObjectInfo{}, nil, err + } + return c.objects.OpenManifest(ctx, identity, id) +} +func (c *pgRawSyncCustody) CopyObject(ctx context.Context, tenant string, ref rawsync.ObjectRef, w io.Writer) (rawsync.ObjectInfo, error) { + if c.tenant != "" && tenant != c.tenant { + return rawsync.ObjectInfo{}, rawsync.ErrUnauthorized + } + if _, err := c.openService(ctx); err != nil { + return rawsync.ObjectInfo{}, err + } + return c.objects.CopyObject(ctx, tenant, ref, w) +} diff --git a/cmd/agentsview/transport.go b/cmd/agentsview/transport.go index abc45658b7..f68329caca 100644 --- a/cmd/agentsview/transport.go +++ b/cmd/agentsview/transport.go @@ -83,6 +83,20 @@ var openPGReadStore = func( pgCfg config.PGConfig, ) (db.Store, func(), error) { applyClassifierConfig(cfg) + if err := pgCfg.ValidateRawDerivation(cfg.RequireAuth); err != nil { + return nil, nil, err + } + if pgCfg.RawTenant != "" { + store, err := postgres.NewHostedStore(pgCfg.URL, pgCfg.Schema, pgCfg.RawTenant, pgCfg.AllowInsecure) + if err != nil { + return nil, nil, err + } + if err = applyRequiredCursorSecret(store, cfg); err != nil { + store.Close() + return nil, nil, err + } + return store, func() { _ = store.Close() }, nil + } store, err := postgres.NewStore( pgCfg.URL, pgCfg.Schema, pgCfg.AllowInsecure, ) diff --git a/docs/hosted-raw-sync.md b/docs/hosted-raw-sync.md index 292995accd..37d9f3969c 100644 --- a/docs/hosted-raw-sync.md +++ b/docs/hosted-raw-sync.md @@ -1,4 +1,5 @@ --- +last_edited: 2026-09-11 title: Hosted Raw Sync description: Keep original session files in hosted custody with authenticated, resumable uploads --- @@ -11,48 +12,230 @@ across restarts. `agentsview raw-sync watch` keeps the hosted copy current. ```mermaid flowchart LR Watcher["Laptop watcher"] -->|"authenticated raw upload"| Custody["Immutable raw custody"] - Custody -. "future" .-> Parser["Server parsing"] + Custody --> Parser["Isolated server parsing"] Parser --> PostgreSQL["PostgreSQL projection"] - PostgreSQL --> Embeddings["Server embeddings"] + PostgreSQL -. "future consumer" .-> Embeddings["Server embeddings"] ``` -The raw archive gives an operator the source material needed to rebuild derived -data. Version 0.42.0 ships capture and upload; it does not yet parse accepted -generations into hosted sessions or build server-owned embeddings. +The server can now parse accepted generations directly into PostgreSQL. Enable +`raw_derivation` on an explicitly provisioned hosted tenant to make uploaded +sessions browsable. Hosted processing uses no SQLite archive intermediary; +SQLite databases captured from providers remain valid source artifacts. +Embedding work is durably queued, but its consumer is not implemented. + +Device enrollment and revocation remain operator-managed. The operator supplies +each laptop with a server URL, device ID and credential. There is no public +enrollment command or HTTP endpoint. The broader delivery work remains tracked +in [issue #1352](https://github.com/kenn-io/agentsview/issues/1352), including +embedding consumption, retention, garbage collection and disaster rebuilds. + +## Provision a hosted instance + +Use one PostgreSQL schema, restricted runtime role and server instance per +tenant. Requests cannot select arbitrary tenants. The schema is permanently +bound to its tenant; runtime connections check the binding, forced row-level +security, constraints, indexes and protected catalog before serving or leasing +work. Provisioning and upgrades require a separate schema-owner connection. + +Configure an owner target and a runtime target in the operator's protected +configuration. Supply actual connection URLs and a generated cursor secret +through your secret manager or protected config file. The values below are +placeholders, not environment-variable interpolation. Use the same tenant and +schema for both targets, and make the runtime target the effective default: + +```toml +default_pg = "hosted" +require_auth = true +cursor_secret = "REPLACE_WITH_BASE64_RANDOM_SECRET" + +[pg.provision] +url = "postgres://hosted_owner@db.example.com/agentsview?sslmode=require" +schema = "hosted_sessions" +raw_tenant = "tenant-example" + +[pg.hosted] +url = "postgres://hosted_runtime@db.example.com/agentsview?sslmode=require" +schema = "hosted_sessions" +raw_tenant = "tenant-example" +raw_derivation = true +raw_poll_seconds = 5 +raw_attempt_seconds = 60 +raw_max_attempts = 5 +``` + +`cursor_secret` is a stable base64-encoded secret shared by restarts of this +instance. Keep authentication enabled even on loopback. Supply TLS through your +reverse proxy and configure the exact public origin as for ordinary remote +access. The shared server bearer token protects viewer APIs; device credentials +and scoped tokens separately protect raw-sync routes. + +Run explicit provisioning with the owner target: + +```bash +agentsview pg hosted-provision provision +``` -!!! note "You need provisioned device credentials" +This command installs or upgrades hosted tables and protections. It does not +create login roles or grant runtime access. Existing derived rows are retained; +existing raw rows must already belong to the chosen tenant. Unknown relations, +unmanaged vector layouts and conflicting ownership can block adoption. Plan +imports and schema upgrades during an operator-controlled maintenance window. +Runtime startup never migrates or provisions the hosted schema. + +Create a separate login role with `NOSUPERUSER NOBYPASSRLS NOCREATEDB +NOCREATEROLE NOREPLICATION NOINHERIT`, provision its credential through your +normal PostgreSQL administration process, and grant it `CONNECT` to the +database. It must own no application objects, have no role memberships, database +or schema `CREATE`, sibling-schema data access, or callable application +`SECURITY DEFINER` functions. Remove inherited `PUBLIC` grants where necessary, +including `CREATE` on the public schema on older PostgreSQL installations. Do +not grant `TRUNCATE`, `TRIGGER` or `REFERENCES` on hosted tables. + +For a newly provisioned schema, the following grants cover full and transcript +content, custody, publication, bounded reparse and supported curation. Replace +`hosted_sessions` and `hosted_runtime` with your schema and restricted role. Run +this as the owner before starting the runtime: - The laptop command is ready to use once the hosted deployment operator gives you - a server URL, device ID, and device credential. Device enrollment and revocation - are operator-managed; AgentsView does not yet provide a public enrollment - command or HTTP endpoint. +```sql +GRANT USAGE ON SCHEMA hosted_sessions TO hosted_runtime; +GRANT SELECT ON ALL TABLES IN SCHEMA hosted_sessions TO hosted_runtime; + +GRANT INSERT ON hosted_sessions.raw_device_tokens, + hosted_sessions.raw_manifest_entries, hosted_sessions.raw_manifest_objects, + hosted_sessions.messages, hosted_sessions.tool_calls, + hosted_sessions.tool_result_events, hosted_sessions.usage_events, + hosted_sessions.secret_findings, hosted_sessions.excluded_sessions, + hosted_sessions.raw_projection_generations, + hosted_sessions.raw_source_contributions, + hosted_sessions.raw_session_public_aliases, + hosted_sessions.raw_embedding_outbox TO hosted_runtime; + +GRANT INSERT, UPDATE ON hosted_sessions.raw_objects, + hosted_sessions.raw_source_heads, hosted_sessions.raw_ingest_jobs, + hosted_sessions.raw_source_projections, hosted_sessions.raw_session_groups, + hosted_sessions.raw_content_revisions, hosted_sessions.raw_session_branches, + hosted_sessions.raw_corpus_state, hosted_sessions.raw_projection_rollouts + TO hosted_runtime; +GRANT INSERT ON hosted_sessions.raw_manifests TO hosted_runtime; + +GRANT INSERT, UPDATE, DELETE ON hosted_sessions.raw_upload_sessions, + hosted_sessions.sessions, hosted_sessions.session_sources, + hosted_sessions.pinned_messages, hosted_sessions.raw_curation, + hosted_sessions.raw_pins TO hosted_runtime; +GRANT INSERT, DELETE ON hosted_sessions.starred_sessions, + hosted_sessions.raw_session_links TO hosted_runtime; + +GRANT USAGE ON SEQUENCE hosted_sessions.raw_ingest_jobs_id_seq, + hosted_sessions.tool_calls_id_seq, hosted_sessions.tool_result_events_id_seq, + hosted_sessions.usage_events_id_seq, hosted_sessions.pinned_messages_id_seq + TO hosted_runtime; +``` - Use [`agentsview pg push`](/docs/pg-sync/) when the shared server must provide - browsable sessions today. It parses sessions locally and can build embeddings - locally before pushing derived rows and vectors to PostgreSQL. +Sequence names above are those created by provisioning. For an adopted schema, +resolve the actual owned sequence with `pg_get_serial_sequence`; serial IDs need +`USAGE` or `UPDATE`, while identity-generated IDs need no separate sequence +grant. Enrollment needs an operator credential with device-write privileges; the +runtime grants intentionally omit those privileges. -The tracked delivery sequence and production acceptance criteria live in -[GitHub issue #1352](https://github.com/kenn-io/agentsview/issues/1352). +`archive_content = "usage"` also requires `SELECT` on `vector_generations` and +`SELECT, DELETE` on existing `vector_documents`, `vector_push_state` and each +existing `vector_chunks_g` table named by a generation. This removes any +previously retained indexed content. It does not create or consume embeddings. +The runtime checks the configured policy's grants before readiness and reports +missing privileges instead of silently disabling hosted processing. -## Delivery status +Start the effective default target: + +```bash +agentsview pg serve --no-browser +``` + +Named targets retain their ordinary selection rules. A single-target deployment +can put the same hosted keys under `[pg]`. `AGENTSVIEW_PG_URL` and +`AGENTSVIEW_PG_SCHEMA` override only the effective default target. They do not +rewrite the separate named owner target. `pg push` refuses a hosted-owned schema +before mutation, even if the client omits its hosted configuration fields. Use a +separate legacy schema for local pushes. + +## Isolation and processing limits + +Hosted parser activation requires Linux amd64 or arm64, a cgo-enabled build, +user/mount/network namespaces, `close_range`, and seccomp with thread +synchronization. Startup tests actual source visibility inside the sandbox +before claiming a job. Unsupported kernels, containers, non-Linux hosts and +Linux builds without cgo fail closed. There is no in-process parser fallback. +The positive kernel suite has been executed on amd64; arm64 has compile proof +but still needs an execution gate on that architecture. + +Each parser child gets only bounded protocol pipes and a minimal environment. A +pre-runtime constructor closes inherited descriptors above stderr before Go +initializes. The child sees a read-only source mount inside an otherwise empty, +read-only jail. Filesystem escape, networking and process creation are denied; +seccomp applies to every existing thread and allows only constrained runtime +thread creation. No external sandbox helper is required. + +| Limit | Value | +| -------------------------------------------- | ----------------------------------------------- | +| Worker concurrency | One sequential job per instance | +| Poll interval | Default 5 seconds; maximum 60 | +| Whole attempt wall time | Default 60 seconds; maximum 300 | +| Attempts per selected generation | Default 5; maximum 10 | +| Retry backoff | Exponential from 1 second, capped at 60 seconds | +| Lease / heartbeat | 60 seconds / 10 seconds | +| Materialized source bytes | 512 MiB | +| Parser stdout / stderr | 32 MiB / 64 KiB | +| Child address space / data | 2 GiB / 512 MiB | +| Child CPU / open descriptors | 30 seconds / 64 | +| Manifest bytes / entries / object references | 1 MiB / 4,096 / 16,384 | + +Custody may accept files larger than the materialization limit. Such captures +remain retained but cannot be projected by this worker. Accepted source bytes +are untrusted, even after device authentication. Publication atomically writes +normalized rows, source proof, identity changes, curation and job outcome under +the selected generation and lease fence. Partial results retain older proof for +unresolved members and retry finitely. Exhausted jobs remain failed until new +source or processing-version selection provides new work. + +Equivalent content from several devices coalesces when it shares a stable +provider session identity. Without that identity, sources remain separate even +when their content matches. Divergent content makes the +bare session ID ambiguous and exposes explicit variants. Source removal retracts +only that source's proof. Names, stars and pins survive compatible publication; +ambiguous identity never silently picks a transcript. Owner imports that change +legacy identity must retry their transaction on serialization failure (`SQLSTATE +40001`) if publication or curation holds a conflicting identity lock. Ordinary +legacy content and curation writes do not take those identity locks. + +The maintenance pass examines at most 64 indexed pending signal rows with a +10-second timeout. It does not scan the archive during idle polling. Shutdown +cancels and joins worker, materializer and parser work before closing custody +and PostgreSQL. + +## Reparse and rollback + +After an executable upgrade changes the parser data version, schedule current +heads explicitly in bounded batches: + +```bash +agentsview pg raw-reparse hosted --run-id parser-rollout-1 --batch-size 64 +``` -| Layer | Status | Current boundary | -| ---------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| Raw custody | Available | Validated objects, canonical manifests, durable receipts, source-head fencing, and parse-job creation | -| Device authentication | Available | Credential exchange, scoped short-lived tokens, server-derived identity, and revocation; enrollment remains operator-managed | -| HTTP raw transport | Available | Missing-object negotiation, resumable upload, and manifest commit; status is local only | -| Laptop capture | Available | Watching, bounded audits, safe SQLite snapshots, durable spooling, checkpoints, retries, and local status | -| Server derivation | Not available | Accepted generations are not yet parsed into PostgreSQL sessions or embeddings | -| Operations and cutover | Not available | Retention, garbage collection, disaster rebuilds, and migration from `pg push` remain future work | +Repeat the same run ID until the command output contains `complete=true`. A call +selects at most 1–256 heads and atomically saves its keyset checkpoint. The run +ID is bound to the executable's processing version. Equal manifest/version +selection is idempotent: a new run ID does not resurrect completed or exhausted +jobs for that same selection. Startup and idle polls perform no reparse scan. -The server parse-worker foundation now includes fenced PostgreSQL job leases, -verified source materialization, provider parsing, retry handling, and a -projection interface. It is an internal library: `pg serve` does not start a -worker, and a PostgreSQL session-projection implementation is still pending. -Hosted browsing and embeddings therefore continue to require `pg push`. +To stop derivation, set `raw_derivation = false` and restart. Keep `raw_tenant`, +authentication, the cursor secret and the tenant-bound runtime connection. +Hosted public reads and raw custody remain available; accepted new manifests +still select their current processing version. Removing `raw_tenant` from an +owned schema fails rather than exposing physical storage identities. This +rollback does not convert the schema back to a `pg push` destination. -The broader delivery issue remains open because public enrollment, hosted -session derivation, and production lifecycle controls are not finished. +Keep PostgreSQL metadata and the immutable raw repository together in backups. +Automated retention, garbage collection, disaster rebuilds, enrollment UX, +embedding consumption and migration cutover tooling remain outside this release. ## Laptop raw watch daemon @@ -82,10 +265,11 @@ doing so intentionally creates two watchers over the same provider roots. ## HTTP control plane -`agentsview pg serve` registers the raw-sync routes when its PostgreSQL role can -write every raw-sync table and the ingest-job sequence. A read-only role keeps -serving the normal PostgreSQL-backed UI and API without these runtime routes. -There is no separate raw-sync configuration switch. When requirements are +In legacy mode, `agentsview pg serve` registers raw-sync routes when its +PostgreSQL role has the required custody-table and ingest-job sequence grants. A +read-only role keeps serving the PostgreSQL UI and API without these routes. +Explicit hosted mode uses `raw_tenant` and requires the full hosted preflight; +`raw_derivation` controls its worker. When legacy route requirements are missing, startup logs `raw-sync routes disabled; missing requirements:` followed by the exact missing table privileges, sequence access, or read-only transaction setting. @@ -191,10 +375,20 @@ The implemented foundations isolate object and metadata identities by tenant and do not deduplicate across tenants. A production deployment must also provide TLS in transit, encryption at rest for object storage, PostgreSQL, backups, and worker scratch space, plus access controls around device enrollment and -revocation. PostgreSQL row-level security remains a planned defense-in-depth -layer; the current foundation does not configure it. +revocation. Explicit hosted provisioning installs forced PostgreSQL row-level +security, tenant constraints and schema binding; runtime validation rejects +weakened protections. This is one tenant per instance, not request-multiplexed +tenancy. Treat the HTTP routes as the protocol between the bundled laptop client and a -hosted AgentsView deployment, not as a general integration API. Public operator -controls, compatibility policy, and recovery tooling will be documented when -those entry points exist. +hosted AgentsView deployment, not as a general integration API. Enrollment and +lifecycle controls remain operator-managed; the provisioning and bounded reparse +commands above are the implemented operator entry points. + +Explicit parent and tool-subagent links first use their own source's historical +proof. Removed or excluded same-source proof prevents fallback. When no such +proof exists, a link may resolve across files only within the same tenant, +device, provider, and configured root, and only to one eligible content cohort. +Conflicting candidates remain unresolved. Fresh session, timing, and sidebar +reads use this rule. A target-only graph change does not necessarily notify an +unchanged owner's session stream; graph-only live refresh is not guaranteed. diff --git a/docs/internal/session-format-sources.md b/docs/internal/session-format-sources.md index 59f9cb63d8..c2d0f32340 100644 --- a/docs/internal/session-format-sources.md +++ b/docs/internal/session-format-sources.md @@ -1,3 +1,7 @@ +--- +last_edited: 2026-09-11 +--- + # Session Format Source Inventory This inventory records the best reproducible evidence currently available for @@ -101,6 +105,11 @@ add an archived or maintained mirror without replacing the original identity. ## Claude Code (`claude`) +Rechecked 2026-09-11 against the existing provider parser and its metadata +fixtures: the first nonempty JSONL `sessionId` supplies `SourceSessionID`. A +filename alone does not supply that provider identity. Hosted multi-device +fixtures retain this field; missing identities remain source-local. + - **Performance fixture check (2026-09-04):** Rechecked the pinned Codeburn format notes below for project-scoped JSONL. `cmd/perfsim` uses the shared Claude fixture builder to emit user/assistant pairs with message/request @@ -268,7 +277,12 @@ add an archived or maintained mirror without replacing the original identity. background flag leave lineage unresolved when their complete UUID sets differ. Identical sets elect the smallest stem, retaining one copy across three background transcripts. An interactive original still wins a tie with - a background sibling. Reverified 2026-08-16 with Claude Code 2.1.233 using a + a background sibling. Reverified 2026-09-11 with same-size sibling rewrites + that restore mtime: filesystem ctime can also remain unchanged. Cached head + metadata therefore requires a bounded leading-byte digest check before + reusing the root UUID or background stamp. Full parsing retains the verified + parent and trims only the replayed prefix; no provider format changed. + Reverified 2026-08-16 with Claude Code 2.1.233 using a controlled `claude -p --session-id ` probe under an isolated `CLAUDE_CONFIG_DIR`. Before the deliberately bounded probe was terminated during its API retry, Claude had created the exact UUID transcript under @@ -497,6 +511,12 @@ add an archived or maintained mirror without replacing the original identity. orphaned child's full transcript when its named parent is unavailable, matching local parsing. When available, the explicitly named parent travels with the captured fork so hosted parsing applies the local replay boundary. + Rechecked 2026-09-11 against the pinned protocol's `forked_from_id` field + and synthetic registered-provider fixtures: an absent parent reports + `DataVersionNeedsRetry` while retaining the child messages; a later captured + readable turnless parent resolves the same child as current. The hosted + integration exercises finite retry exhaustion and new-generation recovery + without changing these provider semantics. Reverified on 2026-09-10 with `TestProviderParserHostedParseMatchesLocalCodexForkLineage`: parents in other configured homes, archives, and custom roots also travel with the @@ -735,7 +755,13 @@ add an archived or maintained mirror without replacing the original identity. available; monetary cost is catalog-derived. - **Agentsview:** `internal/parser/gemini.go` and `internal/parser/gemini_provider.go`; both JSON and JSONL generations remain - supported. + supported. Reverified the pinned recording source on 2026-09-11: token + metadata is attached separately from message content; its writer emits zero + defaults when usage arrives. Hosted transport preserves the parser’s explicit + coverage state, including absent usage and partially populated older records, + rather than inferring coverage from normalized zero-valued keys. The + provider-wire-preparation regression is + `TestSandboxGeminiWirePreservesTokenCoverage`. ## Gemini Apps (`gemini-apps`) diff --git a/docs/pg-sync.md b/docs/pg-sync.md index dc8641d8cb..349c96e964 100644 --- a/docs/pg-sync.md +++ b/docs/pg-sync.md @@ -1,4 +1,5 @@ --- +last_edited: 2026-09-11 title: PostgreSQL Sync description: Share sessions across machines with PostgreSQL push sync, an auto-push service, and a read-only server --- @@ -21,9 +22,28 @@ dashboard as well. [Hosted Raw Sync](/docs/hosted-raw-sync/) is a separate path that keeps original provider artifacts in hosted custody through authenticated, resumable uploads. - It does not yet turn accepted generations into hosted sessions or embeddings. - This does not change `pg push`, `pg push --watch`, or the read-only session UI - and APIs documented on this page. + Explicitly provisioned tenants can enable `raw_derivation` to parse accepted + sources directly into browsable PostgreSQL sessions, without a local SQLite + archive. Embedding consumption remains future work. Ordinary `pg push` and + read-only PostgreSQL serving keep their existing behavior; `pg push` refuses + a schema that has been adopted for hosted raw processing. + +## Hosted raw processing + +For uploads that the server parses itself, provision a schema and tenant with +`agentsview pg hosted-provision `, then serve with a separate +restricted runtime role. The selected PG target needs `raw_tenant`, an explicit +schema and `raw_derivation = true`; the server also requires authentication, a +stable cursor secret and a supported Linux/cgo sandbox. Runtime startup checks +schema protections and grants but never runs hosted migrations. + +See [Hosted Raw Sync](/docs/hosted-raw-sync/#provision-a-hosted-instance) for +the configuration, least-privilege grants and platform limits. Use `agentsview +pg raw-reparse --run-id --batch-size 64` for bounded +parser-version rollouts. To stop the worker, set `raw_derivation = false` and +keep `raw_tenant`, authentication and the cursor secret. This preserves hosted +public reads and raw custody. It does not restore `pg push` access to the owned +schema. ## Quick Start @@ -45,7 +65,9 @@ For multiple PostgreSQL destinations, use named `[pg.NAME]` blocks and `default_pg` instead of the legacy single `[pg]` block. Named target names are normalized case-insensitively, and `all`, `local`, plus the legacy `[pg]` field names `url`, `schema`, `machine_name`, `allow_insecure`, `projects`, and -`exclude_projects` are unavailable as `[pg.NAME]` names. +`exclude_projects`, `raw_tenant`, `raw_derivation`, `raw_poll_seconds`, +`raw_attempt_seconds`, and `raw_max_attempts` are unavailable as `[pg.NAME]` +names. ### 2. Push Sessions @@ -317,7 +339,8 @@ to run it. ### `agentsview pg serve` -Start a read-only web UI backed by PostgreSQL. +Start the PostgreSQL web UI. Legacy mode reads pushed sessions; an explicitly +provisioned hosted target can also derive sessions from accepted raw captures. ```bash agentsview pg serve [flags] @@ -358,14 +381,14 @@ GRANT SELECT, UPDATE ON agentsview.raw_ingest_jobs TO raw_sync_runtime; Run this as the schema owner, substitute your schema and runtime role, and restart `pg serve`. Keep the existing `INSERT` and ingest-job sequence `USAGE` -grants. This enables the raw-sync HTTP routes; `pg serve` does not yet start the -internal parse worker or project hosted raw captures into browsable sessions. +grants. This enables legacy raw-custody HTTP routes. For server-owned parsing, +use the explicit hosted setup below. -On startup, `pg serve` automatically applies any pending schema migrations to -PostgreSQL, creating new tables and indexes added in newer AgentsView versions. -This removes the need to run `pg push` before starting the server after an -upgrade. If the PostgreSQL role is read-only, the migration is skipped and the -server falls back to the schema compatibility check. +In legacy mode, `pg serve` automatically applies any pending schema migrations +to PostgreSQL, creating new tables and indexes added in newer AgentsView +versions. This removes the need to run `pg push` before starting the server +after an upgrade. If the PostgreSQL role is read-only, the migration is skipped +and the server falls back to the schema compatibility check. When `require_auth` is enabled, a bearer token is generated if needed and printed on startup. Pass it via `Authorization: Bearer ` on API requests. @@ -586,7 +609,7 @@ ______________________________________________________________________ are not deleted from PostgreSQL because the local rows no longer exist at push time. Use a direct SQL DELETE to clean up PostgreSQL if needed. Soft-deleted sessions (trash) sync correctly. -- **Schema compatibility** — `pg serve` automatically applies pending schema +- **Schema compatibility** — legacy `pg serve` automatically applies pending schema migrations on startup. If the PostgreSQL role lacks DDL permissions, run `agentsview pg push` from a machine with write access to update the schema. - **Trigram index bloat on pre-0.33.0 schemas** — the content search index was diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 71c3ee6a41..358d212665 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -1726,6 +1726,7 @@ "pinned_expand": "Expand", "pinned_copy_message": "Copy message", "pinned_copied_message": "Copied message", + "pinned_unresolved": "Pinned message is no longer in this transcript.", "pinned_unpin": "Unpin", "settings_session_providers_title": "Session Providers", "settings_session_providers_description": "Choose which agent session sources AgentsView discovers and syncs.", diff --git a/frontend/messages/fr.json b/frontend/messages/fr.json index 99a955880a..1e71ebaa9a 100644 --- a/frontend/messages/fr.json +++ b/frontend/messages/fr.json @@ -1725,6 +1725,7 @@ "pinned_expand": "Déplier", "pinned_copy_message": "Copier le message", "pinned_copied_message": "Message copié", + "pinned_unresolved": "Le message épinglé ne figure plus dans cette conversation.", "pinned_unpin": "Désépingler", "settings_session_providers_title": "Fournisseurs de sessions", "settings_session_providers_description": "Choisissez les sources de sessions d’agents qu’AgentsView détecte et synchronise.", diff --git a/frontend/messages/ja.json b/frontend/messages/ja.json index ddd02bba9b..802f35078c 100644 --- a/frontend/messages/ja.json +++ b/frontend/messages/ja.json @@ -1726,6 +1726,7 @@ "pinned_expand": "展開する", "pinned_copy_message": "メッセージをコピーする", "pinned_copied_message": "コピーされたメッセージ", + "pinned_unresolved": "ピン留めしたメッセージはこの会話に存在しません。", "pinned_unpin": "ピン留めを外す", "settings_session_providers_title": "セッションプロバイダー", "settings_session_providers_description": "AgentsView が検出して同期するエージェント セッション ソースを選択します。", diff --git a/frontend/messages/ko.json b/frontend/messages/ko.json index cd8cf0f040..f6a8a29891 100644 --- a/frontend/messages/ko.json +++ b/frontend/messages/ko.json @@ -1664,6 +1664,7 @@ "pinned_expand": "펼치기", "pinned_copy_message": "메시지 복사", "pinned_copied_message": "메시지가 복사됨", + "pinned_unresolved": "고정한 메시지가 이 대화에 더 이상 없습니다.", "pinned_unpin": "고정 해제", "settings_session_providers_title": "세션 공급자", "settings_session_providers_description": "AgentsView가 검색하고 동기화할 에이전트 세션 소스를 선택합니다.", diff --git a/frontend/messages/zh-CN.json b/frontend/messages/zh-CN.json index cbc6e983f6..1d99efcdc4 100644 --- a/frontend/messages/zh-CN.json +++ b/frontend/messages/zh-CN.json @@ -1662,6 +1662,7 @@ "pinned_expand": "展开", "pinned_copy_message": "复制消息", "pinned_copied_message": "已复制消息", + "pinned_unresolved": "已固定的消息已不在此会话中。", "pinned_unpin": "取消固定", "settings_session_providers_title": "会话提供商", "settings_session_providers_description": "选择 AgentsView 要发现和同步的代理会话来源。", diff --git a/frontend/messages/zh-TW.json b/frontend/messages/zh-TW.json index 0663332de2..1e4bf59dca 100644 --- a/frontend/messages/zh-TW.json +++ b/frontend/messages/zh-TW.json @@ -1662,6 +1662,7 @@ "pinned_expand": "展開", "pinned_copy_message": "複製訊息", "pinned_copied_message": "已複製訊息", + "pinned_unresolved": "已釘選的訊息已不在此對話中。", "pinned_unpin": "取消釘選", "settings_session_providers_title": "工作階段提供者", "settings_session_providers_description": "選擇 AgentsView 要探索及同步的代理工作階段來源。", diff --git a/frontend/src/lib/api/generated/models/apiErrorResponse.ts b/frontend/src/lib/api/generated/models/apiErrorResponse.ts index 5779ad44ce..4862a300f0 100644 --- a/frontend/src/lib/api/generated/models/apiErrorResponse.ts +++ b/frontend/src/lib/api/generated/models/apiErrorResponse.ts @@ -8,5 +8,7 @@ export interface ApiErrorResponse { current_manifest_id?: string; current_receipt?: string; error: string; + state?: string; upload_offset?: number; + variants?: string[]; } diff --git a/frontend/src/lib/api/generated/models/configPGConfig.ts b/frontend/src/lib/api/generated/models/configPGConfig.ts index cec70bb44c..6a10dfff52 100644 --- a/frontend/src/lib/api/generated/models/configPGConfig.ts +++ b/frontend/src/lib/api/generated/models/configPGConfig.ts @@ -8,6 +8,11 @@ export interface ConfigPGConfig { machine_name: string; projects?: string[]; push_vectors?: boolean; + raw_attempt_seconds?: number; + raw_derivation?: boolean; + raw_max_attempts?: number; + raw_poll_seconds?: number; + raw_tenant?: string; schema: string; url: string; } diff --git a/frontend/src/lib/api/generated/models/dbPinnedMessage.ts b/frontend/src/lib/api/generated/models/dbPinnedMessage.ts index 104cd58ce0..f08bc83f50 100644 --- a/frontend/src/lib/api/generated/models/dbPinnedMessage.ts +++ b/frontend/src/lib/api/generated/models/dbPinnedMessage.ts @@ -7,6 +7,7 @@ export interface DbPinnedMessage { created_at: string; id: number; message_id: number; + message_key?: string; note?: string; ordinal: number; role?: string; @@ -15,4 +16,5 @@ export interface DbPinnedMessage { session_first_message?: string; session_id: string; session_project?: string; + unresolved?: boolean; } diff --git a/frontend/src/lib/api/generated/models/dbSession.ts b/frontend/src/lib/api/generated/models/dbSession.ts index 37071b614a..e7f1c3dc24 100644 --- a/frontend/src/lib/api/generated/models/dbSession.ts +++ b/frontend/src/lib/api/generated/models/dbSession.ts @@ -42,6 +42,7 @@ export interface DbSession { outcome: string; outcome_confidence: string; parent_session_id?: string; + parent_session_ids?: string[]; parser_malformed_lines?: number; peak_context_tokens: number; project: string; diff --git a/frontend/src/lib/api/generated/models/dbSidebarSessionIndexRow.ts b/frontend/src/lib/api/generated/models/dbSidebarSessionIndexRow.ts index 7ed28bac72..911e179345 100644 --- a/frontend/src/lib/api/generated/models/dbSidebarSessionIndexRow.ts +++ b/frontend/src/lib/api/generated/models/dbSidebarSessionIndexRow.ts @@ -16,6 +16,7 @@ export interface DbSidebarSessionIndexRow { machine: string; message_count: number; parent_session_id?: string; + parent_session_ids?: string[]; project: string; relationship_type?: string; session_kind?: string; diff --git a/frontend/src/lib/api/generated/models/deleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters.ts b/frontend/src/lib/api/generated/models/deleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters.ts new file mode 100644 index 0000000000..89b1390984 --- /dev/null +++ b/frontend/src/lib/api/generated/models/deleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters.ts @@ -0,0 +1,8 @@ +/** + * Generated by Orval. Do not edit manually. + */ + +export type DeleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters = { + id: string; + messageKey: string; +}; diff --git a/frontend/src/lib/api/generated/models/index.ts b/frontend/src/lib/api/generated/models/index.ts index 534b1be1f4..0d06f97982 100644 --- a/frontend/src/lib/api/generated/models/index.ts +++ b/frontend/src/lib/api/generated/models/index.ts @@ -166,6 +166,7 @@ export * from "./deleteApiV1InsightsByIdPathParameters.ts"; export * from "./deleteApiV1SessionsByIdMessagesByMessageIdPinPathParameters.ts"; export * from "./deleteApiV1SessionsByIdPathParameters.ts"; export * from "./deleteApiV1SessionsByIdPermanentPathParameters.ts"; +export * from "./deleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters.ts"; export * from "./deleteApiV1SessionsByIdStarPathParameters.ts"; export * from "./deleteApiV1SettingsWorktreeMappingsByIdPathParameters.ts"; export * from "./embeddingsBuildRequest.ts"; diff --git a/frontend/src/lib/api/generated/models/serviceSessionDetail.ts b/frontend/src/lib/api/generated/models/serviceSessionDetail.ts index 9dc7e9f8aa..7b213cdd1d 100644 --- a/frontend/src/lib/api/generated/models/serviceSessionDetail.ts +++ b/frontend/src/lib/api/generated/models/serviceSessionDetail.ts @@ -46,6 +46,7 @@ export interface ServiceSessionDetail { outcome: string; outcome_confidence: string; parent_session_id?: string; + parent_session_ids?: string[]; parser_malformed_lines?: number; peak_context_tokens: number; project: string; diff --git a/frontend/src/lib/api/generated/pins/pins.ts b/frontend/src/lib/api/generated/pins/pins.ts index f40bd65bd2..b6547ad1e0 100644 --- a/frontend/src/lib/api/generated/pins/pins.ts +++ b/frontend/src/lib/api/generated/pins/pins.ts @@ -3,6 +3,7 @@ */ import type { DeleteApiV1SessionsByIdMessagesByMessageIdPinPathParameters, + DeleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters, GetApiV1PinsParams, GetApiV1SessionsByIdPinsPathParameters, PinMessageResponse, @@ -94,6 +95,29 @@ export const postApiV1SessionsByIdMessagesByMessageIdPin = async ( ); }; +export const getDeleteApiV1SessionsByIdPinReferencesByMessageKeyUrl = ({ + id, + messageKey, +}: DeleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters) => { + return `/api/v1/sessions/${encodeURIComponent(String(id))}/pin-references/${encodeURIComponent(String(messageKey))}`; +}; + +/** + * @summary Remove retained pin + */ +export const deleteApiV1SessionsByIdPinReferencesByMessageKey = async ( + { id, messageKey }: DeleteApiV1SessionsByIdPinReferencesByMessageKeyPathParameters, + options?: Parameters[1], +): Promise => { + return orvalFetch( + getDeleteApiV1SessionsByIdPinReferencesByMessageKeyUrl({ id, messageKey }), + { + ...options, + method: "DELETE", + }, + ); +}; + export const getGetApiV1SessionsByIdPinsUrl = ({ id }: GetApiV1SessionsByIdPinsPathParameters) => { return `/api/v1/sessions/${encodeURIComponent(String(id))}/pins`; }; diff --git a/frontend/src/lib/api/types/core.ts b/frontend/src/lib/api/types/core.ts index 99f944d03d..43a5277432 100644 --- a/frontend/src/lib/api/types/core.ts +++ b/frontend/src/lib/api/types/core.ts @@ -36,6 +36,7 @@ export interface Session { message_count: number; user_message_count: number; parent_session_id?: string; + parent_session_ids?: string[]; relationship_type?: string; deleted_at?: string | null; termination_status?: string | null; @@ -92,6 +93,7 @@ export interface SessionPage { export interface SidebarSessionIndexRow { id: string; parent_session_id?: string | null; + parent_session_ids?: string[]; relationship_type?: string | null; project: string; machine: string; @@ -197,6 +199,8 @@ export interface AgentsResponse { /** Matches Go PinnedMessage struct in internal/db/pins.go */ export interface PinnedMessage { + message_key?: string; + unresolved?: boolean; id: number; session_id: string; message_id: number; diff --git a/frontend/src/lib/components/pinned/PinnedPage.svelte b/frontend/src/lib/components/pinned/PinnedPage.svelte index dcdc356948..7af46fc9e1 100644 --- a/frontend/src/lib/components/pinned/PinnedPage.svelte +++ b/frontend/src/lib/components/pinned/PinnedPage.svelte @@ -116,7 +116,7 @@ {:else}
- {#each pins.pins as pin (pin.id)} + {#each pins.pins as pin (`${pin.session_id}:${pin.message_key ?? pin.id}`)} {@const info = getSessionInfo(pin)} {@const isExpanded = expanded.has(pin.id)} {@const preview = previewContent(pin.content)} @@ -136,6 +136,9 @@ {formatRelativeTime(pin.created_at)}
+ {#if pin.unresolved} +
{m.pinned_unresolved()}
+ {/if} {#if preview}
{#if isExpanded && pin.content} @@ -154,6 +157,7 @@
diff --git a/frontend/src/lib/stores/pins.svelte.ts b/frontend/src/lib/stores/pins.svelte.ts index 71b0eb1a46..bc3d80aee6 100644 --- a/frontend/src/lib/stores/pins.svelte.ts +++ b/frontend/src/lib/stores/pins.svelte.ts @@ -81,7 +81,9 @@ class PinsStore { this.#loadVersion === loadVer && this.#mutationVersion === mutVer ) { - this.sessionPinIds = new Set(res.pins.map((p) => p.message_id)); + this.sessionPinIds = new Set( + res.pins.filter((p) => !p.unresolved).map((p) => p.message_id), + ); } } catch (e) { if (isAbortError(e) || !this.#sessionPinsRead.isCurrent(signal)) return; @@ -119,6 +121,25 @@ class PinsStore { } } + async removeReference(pin: PinnedMessage) { + if (!pin.message_key) return; + this.#mutationVersion++; + try { + await PinsService.deleteApiV1SessionsByIdPinReferencesByMessageKey({ + id: pin.session_id, + messageKey: pin.message_key, + }); + this.pins = this.pins.filter( + (p) => !(p.session_id === pin.session_id && p.message_key === pin.message_key), + ); + } catch { + // Reload below reconciles failed removal without losing the retained card. + } finally { + this.loadAll(this.#loadedProject); + if (this.#currentSessionId) this.loadForSession(this.#currentSessionId); + } + } + async unpin(sessionId: string, messageId: number) { if (this.#inflight.has(messageId)) return; this.#inflight.add(messageId); diff --git a/frontend/src/lib/stores/pins.test.ts b/frontend/src/lib/stores/pins.test.ts index 7582c97f12..ec7494e579 100644 --- a/frontend/src/lib/stores/pins.test.ts +++ b/frontend/src/lib/stores/pins.test.ts @@ -17,6 +17,7 @@ vi.mock("../api/generated/index", () => ({ getApiV1SessionsByIdPins: vi.fn().mockResolvedValue({ pins: [] }), postApiV1SessionsByIdMessagesByMessageIdPin: vi.fn().mockResolvedValue({ id: 1 }), deleteApiV1SessionsByIdMessagesByMessageIdPin: vi.fn().mockResolvedValue(undefined), + deleteApiV1SessionsByIdPinReferencesByMessageKey: vi.fn().mockResolvedValue(undefined), }, })); @@ -199,3 +200,35 @@ describe("PinsStore.loadAll project filtering", () => { expect(store.pins).toEqual([PIN_BETA]); }); }); + +describe("retained pin references", () => { + it("does not mark a reused ordinal pinned and removes the retained message key", async () => { + const store = createPinsStore(); + const oldPin = { + ...PIN_ALPHA, + message_id: 1, + ordinal: 1, + message_key: "old-message", + unresolved: true, + }; + const currentPin = { + ...PIN_ALPHA, + id: 2, + message_id: 2, + ordinal: 2, + message_key: "current-message", + }; + pinsService.getApiV1SessionsByIdPins.mockResolvedValue({ pins: [oldPin, currentPin] }); + await store.loadForSession("s1"); + expect(store.isPinned(1)).toBe(false); + expect(store.isPinned(2)).toBe(true); + store.pins = [oldPin, currentPin]; + pinsService.getApiV1Pins.mockResolvedValue({ pins: [currentPin] }); + await store.removeReference(oldPin); + expect(PinsService.deleteApiV1SessionsByIdPinReferencesByMessageKey).toHaveBeenCalledWith({ + id: "s1", + messageKey: "old-message", + }); + expect(store.pins.map((p) => p.message_key)).toEqual(["current-message"]); + }); +}); diff --git a/frontend/src/lib/stores/sessions.svelte.ts b/frontend/src/lib/stores/sessions.svelte.ts index c7c82e9ecc..4e03724606 100644 --- a/frontend/src/lib/stores/sessions.svelte.ts +++ b/frontend/src/lib/stores/sessions.svelte.ts @@ -30,6 +30,7 @@ const RECENTLY_DELETED_TTL_MS = 10_000; export interface SessionGroupInput { id: string; parent_session_id?: string | null; + parent_session_ids?: string[]; relationship_type?: string | null; project: string; machine: string; @@ -1543,6 +1544,7 @@ function sidebarIndexRowToSession(row: SidebarSessionIndexRow, existing?: Sessio message_count: row.message_count, user_message_count: row.user_message_count, parent_session_id: row.parent_session_id ?? undefined, + parent_session_ids: row.parent_session_ids, relationship_type: row.relationship_type ?? undefined, termination_status: row.termination_status ?? null, total_output_tokens: 0, @@ -1570,6 +1572,7 @@ function sidebarIndexRowToSession(row: SidebarSessionIndexRow, existing?: Sessio message_count: skinny.message_count, user_message_count: skinny.user_message_count, parent_session_id: skinny.parent_session_id, + parent_session_ids: skinny.parent_session_ids, relationship_type: skinny.relationship_type, termination_status: skinny.termination_status, transcript_revision: skinny.transcript_revision, @@ -1719,6 +1722,30 @@ function findRoot( return cur; } +// A hosted equal-content child may belong to several proven parent cohorts. +// Keep every reachable loaded root, without choosing a representative parent. +function findPublicRoots(id: string, byId: Map): string[] { + const pending = [id]; + const visited = new Set(); + const roots = new Set(); + while (pending.length) { + const current = pending.pop()!; + if (visited.has(current)) continue; + visited.add(current); + const row = byId.get(current); + const parents = ( + row?.parent_session_ids?.length + ? row.parent_session_ids + : row?.parent_session_id + ? [row.parent_session_id] + : [] + ).filter((parent) => byId.has(parent)); + if (!parents.length) roots.add(current); + else pending.push(...parents); + } + return [...roots].sort(); +} + export function buildSessionGroups(sessions: SessionGroupInput[]): SessionGroup[] { const byId = new Map(); for (const s of sessions) { @@ -1729,33 +1756,37 @@ export function buildSessionGroups(sessions: SessionGroupInput[]): SessionGroup[ const groupMap = new Map(); const insertionOrder: string[] = []; + const pluralParents = sessions.some((s) => s.parent_session_ids?.length); for (const s of sessions) { - const root = findRoot(s.id, byId, rootCache); - // Sessions without a parent_session_id that aren't - // pointed to by anyone get root == their own id, so - // they form a single-session group naturally. - const key = root; - - let group = groupMap.get(key); - if (!group) { - group = { - key, - project: s.project, - sessions: [], - primarySessionId: s.id, - totalMessages: 0, - firstMessage: null, - startedAt: null, - endedAt: null, - }; - groupMap.set(key, group); - insertionOrder.push(key); - } + const roots = pluralParents ? findPublicRoots(s.id, byId) : [findRoot(s.id, byId, rootCache)]; + if (!roots.length) roots.push(findRoot(s.id, byId, rootCache)); + for (const root of roots) { + // Sessions without a parent_session_id that aren't + // pointed to by anyone get root == their own id, so + // they form a single-session group naturally. + const key = root; + + let group = groupMap.get(key); + if (!group) { + group = { + key, + project: s.project, + sessions: [], + primarySessionId: s.id, + totalMessages: 0, + firstMessage: null, + startedAt: null, + endedAt: null, + }; + groupMap.set(key, group); + insertionOrder.push(key); + } - group.sessions.push(s); - group.totalMessages += s.message_count; - group.startedAt = minString(group.startedAt, s.started_at); - group.endedAt = maxString(group.endedAt, s.ended_at); + group.sessions.push(s); + group.totalMessages += s.message_count; + group.startedAt = minString(group.startedAt, s.started_at); + group.endedAt = maxString(group.endedAt, s.ended_at); + } } // Adopt orphaned teammate sessions so they NEVER appear at root level. diff --git a/frontend/src/lib/stores/sessions.test.ts b/frontend/src/lib/stores/sessions.test.ts index f0e76c8c0f..e5c6e5287f 100644 --- a/frontend/src/lib/stores/sessions.test.ts +++ b/frontend/src/lib/stores/sessions.test.ts @@ -2903,6 +2903,25 @@ describe("buildSessionGroups", () => { expect(groups[0]!.sessions.map((s) => s.id)).toContain("teammate"); }); + it("includes a shared hosted child under both proven parent variants", () => { + const rows = [ + makeSession({ id: "parent-a", project: "proj" }), + makeSession({ id: "parent-b", project: "proj" }), + { + ...makeSession({ id: "shared-child", project: "proj" }), + parent_session_ids: ["parent-a", "parent-b"], + relationship_type: "subagent", + }, + ]; + const groups = buildSessionGroups(rows); + expect( + groups.map((group) => ({ key: group.key, ids: group.sessions.map((session) => session.id) })), + ).toEqual([ + { key: "parent-a", ids: ["parent-a", "shared-child"] }, + { key: "parent-b", ids: ["parent-b", "shared-child"] }, + ]); + }); + it("groups two-session chain", () => { const sessions = [ makeSession({ diff --git a/internal/activity/reportartifacts.go b/internal/activity/reportartifacts.go index 6caca80993..0c18229509 100644 --- a/internal/activity/reportartifacts.go +++ b/internal/activity/reportartifacts.go @@ -39,6 +39,9 @@ type ProgressFunc func(Progress) // archive changes may force a harmless refresh, while the canonical artifact // digest remains the final consistency check. type SourceProbe struct { + HostedIdentityRevision int64 `json:"hosted_identity_revision,omitempty"` + HostedSelectionRevision int64 `json:"hosted_selection_revision,omitempty"` + HostedCorpusRevision int64 `json:"hosted_corpus_revision,omitempty"` SessionCount int64 `json:"session_count"` MaxSessionModified string `json:"max_session_modified"` MaxDataVersion int64 `json:"max_data_version"` diff --git a/internal/artifact/manifest_session_test.go b/internal/artifact/manifest_session_test.go index 58f6d22f2f..8c8eb3e2d4 100644 --- a/internal/artifact/manifest_session_test.go +++ b/internal/artifact/manifest_session_test.go @@ -27,13 +27,19 @@ func TestManifestSessionMatchesDBSessionWireFormat(t *testing.T) { var sess db.Session populateWireFixture(t, reflect.ValueOf(&sess).Elem(), 1) + require.Len(t, sess.ParentSessionIDs, 2) + require.NotEqual(t, sess.ParentSessionIDs[0], sess.ParentSessionIDs[1]) // Deliberate parity exemption: quality_signals is hoisted to the // manifest-level session_quality_signals field because db.Session's // pointer is load-path-transient (see the manifestSession struct - // comment). The reference for parity is the session without it. + // comment). ParentSessionIDs is also deliberately absent: plural parents + // describe hosted read context, not portable artifact content. Neither + // field may change the pinned artifact bytes. Scalar ParentSessionID stays + // in the reference and retains its existing wire contract. reference := sess reference.QualitySignals = nil + reference.ParentSessionIDs = nil type sessionAlias db.Session want, err := canonicalJSON(sessionAlias(reference)) @@ -41,12 +47,20 @@ func TestManifestSessionMatchesDBSessionWireFormat(t *testing.T) { got, err := canonicalJSON(manifestSessionFromDB(sess)) require.NoError(t, err) assert.Equal(t, string(want), string(got), - "manifestSession must serialize byte-identically to db.Session minus quality_signals") + "manifestSession must serialize byte-identically to db.Session minus transient quality_signals and hosted parent context") - withoutPointer, err := canonicalJSON(manifestSessionFromDB(reference)) + withoutReadMetadata, err := canonicalJSON(manifestSessionFromDB(reference)) require.NoError(t, err) - assert.Equal(t, string(got), string(withoutPointer), - "manifest bytes must not depend on the transient quality_signals pointer") + assert.Equal(t, string(got), string(withoutReadMetadata), + "manifest bytes must not depend on the transient quality_signals pointer or hosted plural parents") + + for _, parents := range [][]string{nil, {}, {"other-parent-a", "other-parent-b"}} { + changed := sess + changed.ParentSessionIDs = parents + wire, err := canonicalJSON(manifestSessionFromDB(changed)) + require.NoError(t, err) + assert.Equal(t, string(got), string(wire), "hosted parent context must not rehash artifacts") + } roundTrip, err := canonicalJSON(sessionAlias(manifestSessionFromDB(sess).dbSession())) require.NoError(t, err) @@ -105,6 +119,12 @@ func setWireFixtureValue(t *testing.T, field reflect.Value, n int) { elem := reflect.New(field.Type().Elem()) setWireFixtureValue(t, elem.Elem(), n) field.Set(elem) + case reflect.Slice: + values := reflect.MakeSlice(field.Type(), 2, 2) + for i := 0; i < values.Len(); i++ { + setWireFixtureValue(t, values.Index(i), n*10+i) + } + field.Set(values) case reflect.Struct: populateWireFixture(t, field, n*10) default: diff --git a/internal/backendcontract/contract.go b/internal/backendcontract/contract.go index e2820abccb..b56477244c 100644 --- a/internal/backendcontract/contract.go +++ b/internal/backendcontract/contract.go @@ -11,5 +11,6 @@ import ( var ( _ db.Store = (*db.DB)(nil) _ db.Store = (*postgresstore.Store)(nil) + _ db.Store = (*postgresstore.HostedStore)(nil) _ db.Store = (*duckdbstore.Store)(nil) ) diff --git a/internal/config/config.go b/internal/config/config.go index 5bdb7136b9..148d729012 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,12 @@ type ProxyConfig struct { // PGConfig holds PostgreSQL connection settings. type PGConfig struct { + RawTenant string `toml:"raw_tenant" json:"raw_tenant,omitempty"` + RawDerivation bool `toml:"raw_derivation" json:"raw_derivation,omitempty"` + RawPollSeconds int `toml:"raw_poll_seconds" json:"raw_poll_seconds,omitempty"` + RawAttemptSeconds int `toml:"raw_attempt_seconds" json:"raw_attempt_seconds,omitempty"` + RawMaxAttempts int `toml:"raw_max_attempts" json:"raw_max_attempts,omitempty"` + URL string `toml:"url" json:"url"` Schema string `toml:"schema" json:"schema"` MachineName string `toml:"machine_name" json:"machine_name"` @@ -97,6 +103,8 @@ type ResolvedPGTarget struct { } var pgConfigKeys = map[string]struct{}{ + "raw_tenant": {}, "raw_derivation": {}, "raw_poll_seconds": {}, "raw_attempt_seconds": {}, "raw_max_attempts": {}, + "url": {}, "schema": {}, "machine_name": {}, diff --git a/internal/config/raw_derivation.go b/internal/config/raw_derivation.go new file mode 100644 index 0000000000..29ed8fd9a9 --- /dev/null +++ b/internal/config/raw_derivation.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "strings" +) + +// ValidateRawDerivation validates opt-in and bounded controls. Tenant binding +// remains enabled with the worker off so rollback preserves public identities. +func (p PGConfig) ValidateRawDerivation(requireAuth bool) error { + if !p.RawDerivation && p.RawTenant == "" { + return nil + } + if strings.TrimSpace(p.RawTenant) == "" || p.RawTenant != strings.TrimSpace(p.RawTenant) || len(p.RawTenant) > 128 { + return fmt.Errorf("hosted raw projection requires raw_tenant") + } + if !requireAuth { + return fmt.Errorf("hosted raw projection requires authentication") + } + if p.Schema == "" { + return fmt.Errorf("hosted raw projection requires a configured schema") + } + for _, r := range p.Schema { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '_' { + return fmt.Errorf("hosted raw projection schema is invalid") + } + } + if p.RawPollSeconds < 0 || p.RawPollSeconds > 60 || p.RawAttemptSeconds < 0 || p.RawAttemptSeconds > 300 || p.RawMaxAttempts < 0 || p.RawMaxAttempts > 10 { + return fmt.Errorf("hosted raw worker bounds: poll 1-60 seconds, attempt 1-300 seconds, attempts 1-10; zero selects defaults") + } + return nil +} +func (p PGConfig) RawWorkerBounds() (poll, attempt, attempts int) { + poll, attempt, attempts = p.RawPollSeconds, p.RawAttemptSeconds, p.RawMaxAttempts + if poll == 0 { + poll = 5 + } + if attempt == 0 { + attempt = 60 + } + if attempts == 0 { + attempts = 5 + } + return +} diff --git a/internal/config/raw_derivation_test.go b/internal/config/raw_derivation_test.go new file mode 100644 index 0000000000..91d4d364ed --- /dev/null +++ b/internal/config/raw_derivation_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRawDerivationConfigBindingAndBounds(t *testing.T) { + for _, tc := range []struct { + name string + p PGConfig + auth bool + valid bool + }{ + {"off", PGConfig{}, false, true}, + {"missing tenant", PGConfig{RawDerivation: true}, true, false}, + {"missing auth", PGConfig{RawTenant: "tenant", RawDerivation: true, Schema: "hosted"}, false, false}, + {"valid", PGConfig{RawTenant: "tenant", RawDerivation: true, Schema: "hosted"}, true, true}, + {"negative poll", PGConfig{RawTenant: "tenant", RawDerivation: true, Schema: "hosted", RawPollSeconds: -1}, true, false}, + {"unbounded attempt", PGConfig{RawTenant: "tenant", RawDerivation: true, Schema: "hosted", RawAttemptSeconds: 301}, true, false}, + {"unbounded retries", PGConfig{RawTenant: "tenant", RawDerivation: true, Schema: "hosted", RawMaxAttempts: 11}, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.p.ValidateRawDerivation(tc.auth) + if tc.valid { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } + legacy, targets, err := parsePGConfigSection(map[string]any{"raw_tenant": "tenant", "raw_derivation": true, "raw_max_attempts": int64(3)}) + require.NoError(t, err) + assert.Empty(t, targets) + assert.Equal(t, "tenant", legacy.RawTenant) + assert.True(t, legacy.RawDerivation) + assert.Equal(t, 3, legacy.RawMaxAttempts) + _, targets, err = parsePGConfigSection(map[string]any{"remote": map[string]any{"raw_tenant": "tenant", "raw_derivation": true}}) + require.NoError(t, err) + assert.True(t, targets["remote"].RawDerivation) + _, _, err = parsePGConfigSection(map[string]any{"raw_tenant": map[string]any{"url": "invalid"}}) + assert.Error(t, err) +} diff --git a/internal/db/archive_content.go b/internal/db/archive_content.go index 20a0669d11..dd3932ebdd 100644 --- a/internal/db/archive_content.go +++ b/internal/db/archive_content.go @@ -78,27 +78,23 @@ func (db *DB) requireDerivedTextStorage(kind string) error { } func (db *DB) sessionForStorage(session Session) Session { - if !db.usageOnlyStorage() { - return session - } - // Derive automation while the parser/importer preview is still present. - // The retained session row is authoritative after transcript text is gone. - session.IsAutomated = sessionIsAutomated(session) - session.PreserveStoredAutomation = session.FirstMessage == nil && - session.UserMessageCount <= 1 - session.FirstMessage = nil - session.DisplayName = nil - session.SessionName = nil - session.PreserveSessionName = false - session.SecretLeakCount = 0 - session.SecretsRulesVersion = "" - return session + return sessionForStoragePolicy(session, db.ArchiveContent()) } func (db *DB) sessionAndMessagesForStorage( session Session, messages []Message, ) (Session, []Message) { - switch db.ArchiveContent() { + return ProjectSessionForStoragePolicy( + session, messages, db.ArchiveContent(), + ) +} + +// ProjectSessionForStoragePolicy applies an explicit storage policy without a +// database handle. Hosted and local ingestion use this same projection. +func ProjectSessionForStoragePolicy( + session Session, messages []Message, policy config.ArchiveContent, +) (Session, []Message) { + switch policy { case config.ArchiveContentUsage: // Some importers do not precompute IsAutomated. Classify from the raw // messages before the storage projection drops user text. @@ -106,7 +102,7 @@ func (db *DB) sessionAndMessagesForStorage( IsAutomatedTranscript( session.UserMessageCount, messages, session.FirstMessage, ) - return db.sessionForStorage(session), usageOnlyMessages(messages) + return sessionForStoragePolicy(session, policy), usageOnlyMessages(messages) case config.ArchiveContentTranscripts: return session, transcriptMessages(messages) default: @@ -114,6 +110,26 @@ func (db *DB) sessionAndMessagesForStorage( } } +func sessionForStoragePolicy( + session Session, policy config.ArchiveContent, +) Session { + if !policy.UsageOnly() { + return session + } + // Derive automation while the parser/importer preview is still present. + // The retained session row is authoritative after transcript text is gone. + session.IsAutomated = sessionIsAutomated(session) + session.PreserveStoredAutomation = session.FirstMessage == nil && + session.UserMessageCount <= 1 + session.FirstMessage = nil + session.DisplayName = nil + session.SessionName = nil + session.PreserveSessionName = false + session.SecretLeakCount = 0 + session.SecretsRulesVersion = "" + return session +} + // ProjectSessionForStorage applies this database handle's storage policy to a // prepared session without writing it. Report-only callers use the same // projection as the write boundary before comparing prepared and stored rows. diff --git a/internal/db/hosted_identity.go b/internal/db/hosted_identity.go new file mode 100644 index 0000000000..2cf8df0117 --- /dev/null +++ b/internal/db/hosted_identity.go @@ -0,0 +1,35 @@ +package db + +import ( + "context" + "fmt" +) + +// SessionIdentityError reports a public alias that cannot select one transcript. +// Variants contains public choices, never physical storage row identities. +type SessionIdentityError struct { + State string `json:"state"` + Variants []string `json:"variants,omitempty"` +} + +func (e *SessionIdentityError) Error() string { return fmt.Sprintf("session identity is %s", e.State) } + +// SessionWatchStateStore lets remote watches notice alias membership transitions. +type SessionWatchStateStore interface { + GetSessionWatchState(id string) (SessionWatchState, error) +} + +const SessionWatchResolved = "resolved" + +type SessionWatchState struct { + State string `json:"state"` + PublicID string `json:"public_id,omitempty"` + Variants []string `json:"variants,omitempty"` + IdentityRevision int64 `json:"identity_revision"` + ContentRevision string `json:"content_revision,omitempty"` +} + +// ProviderResumeIdentityStore converts a hosted alias to provider-owned resume identity. +type ProviderResumeIdentityStore interface { + GetProviderResumeID(ctx context.Context, id string) (string, error) +} diff --git a/internal/db/pins.go b/internal/db/pins.go index 7ffdd07445..7c18d52e73 100644 --- a/internal/db/pins.go +++ b/internal/db/pins.go @@ -8,14 +8,16 @@ import ( // PinnedMessage represents a row in the pinned_messages table. type PinnedMessage struct { - ID int64 `json:"id"` - SessionID string `json:"session_id"` - MessageID int64 `json:"message_id"` - Ordinal int `json:"ordinal"` - Note *string `json:"note,omitempty"` - Content *string `json:"content,omitempty"` - Role *string `json:"role,omitempty"` - CreatedAt string `json:"created_at"` + MessageKey string `json:"message_key,omitempty"` + Unresolved bool `json:"unresolved,omitempty"` + ID int64 `json:"id"` + SessionID string `json:"session_id"` + MessageID int64 `json:"message_id"` + Ordinal int `json:"ordinal"` + Note *string `json:"note,omitempty"` + Content *string `json:"content,omitempty"` + Role *string `json:"role,omitempty"` + CreatedAt string `json:"created_at"` // Session metadata — populated only for the "all pins" query. SessionProject *string `json:"session_project,omitempty"` @@ -232,3 +234,8 @@ func (db *DB) GetPinnedMessageIDs( } return ids, rows.Err() } + +// PinReferenceStore removes a retained normalized anchor after its message disappears. +type PinReferenceStore interface { + RemovePinReference(ctx context.Context, sessionID, messageKey string) error +} diff --git a/internal/db/query_dialect.go b/internal/db/query_dialect.go index 405912d216..871718e236 100644 --- a/internal/db/query_dialect.go +++ b/internal/db/query_dialect.go @@ -28,6 +28,7 @@ const ( // ORM: callers still own SELECTs, JOINs, backend-specific search paths, and // table schemas. type QueryDialect struct { + parentRelation func(childAlias, parentAlias string) string name string placeholderStyle placeholderStyle trueLiteral string @@ -469,6 +470,16 @@ func SidebarOrphanPredicate(sessionAlias, parentAlias string) string { )` } +func (d QueryDialect) WithParentRelation(relation func(string, string) string) QueryDialect { + d.parentRelation = relation + return d +} +func (d QueryDialect) ParentRelation(child, parent string) string { + if d.parentRelation != nil { + return d.parentRelation(child, parent) + } + return child + ".parent_session_id = " + parent + ".id" +} func BuildCanonicalRootWhere(dialect QueryDialect, sessionAlias string, includeOrphans bool) string { base := `NOT (` + CanonicalChildRelationshipPredicate(dialect, sessionAlias) + `)` if !includeOrphans { @@ -476,7 +487,7 @@ func BuildCanonicalRootWhere(dialect QueryDialect, sessionAlias string, includeO } return `(` + base + ` OR (` + CanonicalChildRelationshipPredicate(dialect, sessionAlias) + ` AND ` + - SidebarOrphanPredicate(sessionAlias, "parent") + `))` + `NOT EXISTS (SELECT 1 FROM sessions parent WHERE ` + dialect.ParentRelation(sessionAlias, "parent") + `)` + `))` } func buildSessionFilterWithBuilder( @@ -531,7 +542,7 @@ func buildSessionFilterWithBuilder( rootMatch + " UNION " + "SELECT s.id FROM sessions s" + - " JOIN tree t ON s.parent_session_id = t.id" + + " JOIN tree t ON " + b.dialect.ParentRelation("s", "t") + " WHERE s.message_count > 0 AND s.deleted_at IS NULL" + childAutomationWhere + ") SELECT id FROM tree" diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 7a98f4883c..8f62e6d4f5 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -293,28 +293,29 @@ func (s *Session) UnmarshalJSON(data []byte) error { // Session represents a row in the sessions table. type Session struct { - ID string `json:"id"` - Project string `json:"project"` - Machine string `json:"machine"` - Agent string `json:"agent"` - AgentLabel string `json:"agent_label,omitempty"` - Entrypoint string `json:"entrypoint,omitempty"` - SessionKind string `json:"session_kind,omitempty"` - FirstMessage *string `json:"first_message"` - DisplayName *string `json:"display_name,omitempty"` - SessionName *string `json:"-"` - StartedAt *string `json:"started_at"` - EndedAt *string `json:"ended_at"` - MessageCount int `json:"message_count"` - UserMessageCount int `json:"user_message_count"` - ParentSessionID *string `json:"parent_session_id,omitempty"` - ParserParentSessionID *string `json:"-"` - RelationshipType string `json:"relationship_type,omitempty"` - TotalOutputTokens int `json:"total_output_tokens"` - PeakContextTokens int `json:"peak_context_tokens"` - HasTotalOutputTokens bool `json:"has_total_output_tokens"` - HasPeakContextTokens bool `json:"has_peak_context_tokens"` - IsAutomated bool `json:"is_automated"` + ID string `json:"id"` + Project string `json:"project"` + Machine string `json:"machine"` + Agent string `json:"agent"` + AgentLabel string `json:"agent_label,omitempty"` + Entrypoint string `json:"entrypoint,omitempty"` + SessionKind string `json:"session_kind,omitempty"` + FirstMessage *string `json:"first_message"` + DisplayName *string `json:"display_name,omitempty"` + SessionName *string `json:"-"` + StartedAt *string `json:"started_at"` + EndedAt *string `json:"ended_at"` + MessageCount int `json:"message_count"` + UserMessageCount int `json:"user_message_count"` + ParentSessionIDs []string `json:"parent_session_ids,omitempty"` + ParentSessionID *string `json:"parent_session_id,omitempty"` + ParserParentSessionID *string `json:"-"` + RelationshipType string `json:"relationship_type,omitempty"` + TotalOutputTokens int `json:"total_output_tokens"` + PeakContextTokens int `json:"peak_context_tokens"` + HasTotalOutputTokens bool `json:"has_total_output_tokens"` + HasPeakContextTokens bool `json:"has_peak_context_tokens"` + IsAutomated bool `json:"is_automated"` // Session signals (computed from messages/tool_calls). ToolFailureSignalCount int `json:"tool_failure_signal_count"` @@ -638,25 +639,26 @@ type SessionPage struct { } type SidebarSessionIndexRow struct { - ID string `json:"id"` - ParentSessionID *string `json:"parent_session_id,omitempty"` - RelationshipType string `json:"relationship_type,omitempty"` - Project string `json:"project"` - Machine string `json:"machine"` - Agent string `json:"agent"` - AgentLabel string `json:"agent_label,omitempty"` - Entrypoint string `json:"entrypoint,omitempty"` - SessionKind string `json:"session_kind,omitempty"` - DisplayName *string `json:"display_name,omitempty"` - StartedAt *string `json:"started_at"` - EndedAt *string `json:"ended_at"` - CreatedAt string `json:"created_at"` - TerminationStatus *string `json:"termination_status,omitempty"` - MessageCount int `json:"message_count"` - UserMessageCount int `json:"user_message_count"` - TranscriptRevision *string `json:"transcript_revision,omitempty"` - IsAutomated bool `json:"is_automated"` - IsTeammate bool `json:"is_teammate"` + ParentSessionIDs []string `json:"parent_session_ids,omitempty"` + ID string `json:"id"` + ParentSessionID *string `json:"parent_session_id,omitempty"` + RelationshipType string `json:"relationship_type,omitempty"` + Project string `json:"project"` + Machine string `json:"machine"` + Agent string `json:"agent"` + AgentLabel string `json:"agent_label,omitempty"` + Entrypoint string `json:"entrypoint,omitempty"` + SessionKind string `json:"session_kind,omitempty"` + DisplayName *string `json:"display_name,omitempty"` + StartedAt *string `json:"started_at"` + EndedAt *string `json:"ended_at"` + CreatedAt string `json:"created_at"` + TerminationStatus *string `json:"termination_status,omitempty"` + MessageCount int `json:"message_count"` + UserMessageCount int `json:"user_message_count"` + TranscriptRevision *string `json:"transcript_revision,omitempty"` + IsAutomated bool `json:"is_automated"` + IsTeammate bool `json:"is_teammate"` } type SidebarSessionIndex struct { diff --git a/internal/ingest/convert.go b/internal/ingest/convert.go new file mode 100644 index 0000000000..a8d197eb04 --- /dev/null +++ b/internal/ingest/convert.go @@ -0,0 +1,260 @@ +package ingest + +import ( + "context" + "slices" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/timeutil" +) + +// ConvertSessionContext converts parser session metadata into a storage row. +func ConvertSessionContext( + ctx context.Context, parsed parser.ParsedSession, + messages []parser.ParsedMessage, +) (db.Session, error) { + hasTotal, hasPeak, err := parsed.TokenCoverageContext(ctx, messages) + if err != nil { + return db.Session{}, err + } + parent := stringPtr(parsed.ParentSessionID) + session := db.Session{ + ID: parsed.ID, Project: parsed.Project, Machine: parsed.Machine, + MessageCount: parsed.MessageCount, UserMessageCount: parsed.UserMessageCount, + ParentSessionID: parent, ParserParentSessionID: cloneStringPtr(parent), + RelationshipType: string(parsed.RelationshipType), + TotalOutputTokens: parsed.TotalOutputTokens, + PeakContextTokens: parsed.PeakContextTokens, + HasTotalOutputTokens: hasTotal, HasPeakContextTokens: hasPeak, + Cwd: parsed.Cwd, GitBranch: parsed.GitBranch, + SourceSessionID: parsed.SourceSessionID, + SourceVersion: parsed.SourceVersion, + TranscriptFidelity: parsed.TranscriptFidelity, + ParserMalformedLines: parsed.MalformedLines, + IsTruncated: parsed.IsTruncated, + TerminationStatus: stringPtr(string(parsed.TerminationStatus)), + FilePath: stringPtr(parsed.File.Path), FileSize: int64Ptr(parsed.File.Size), + FileMtime: int64Ptr(parsed.File.Mtime), + NextOrdinal: NextParsedOrdinal(0, messages), + LastEntryUUID: stringPtr(LastParsedSourceUUID("", messages)), + ClaudeLinearParse: parsed.ClaudeLinearParse, + FileInode: int64Ptr(parsed.File.Inode), + FileDevice: int64Ptr(parsed.File.Device), + FileHash: stringPtr(parsed.File.Hash), + } + db.ApplyParsedSessionIdentity(&session, parsed) + if parsed.FirstMessage != "" { + session.FirstMessage = &parsed.FirstMessage + } + session.SessionName = db.ParsedSessionName(parsed) + session.PreserveSessionName = parsed.Agent == parser.AgentCodex && + !parsed.SessionNamePresent + if !parsed.StartedAt.IsZero() { + session.StartedAt = timeutil.Ptr(parsed.StartedAt) + } + if !parsed.EndedAt.IsZero() { + session.EndedAt = timeutil.Ptr(parsed.EndedAt) + } + return session, ctx.Err() +} + +// ConvertMessagesContext converts messages and applies result pairing and +// carrier filtering. +func ConvertMessagesContext( + ctx context.Context, sessionID string, agent parser.AgentType, + parsed []parser.ParsedMessage, blocked map[string]bool, +) ([]db.Message, error) { + messages := make([]db.Message, len(parsed)) + for i, message := range parsed { + if err := ctx.Err(); err != nil { + return nil, err + } + hasContext, hasOutput := message.TokenPresence() + toolCalls, err := ConvertToolCallsContext( + ctx, sessionID, message.ToolCalls, + ) + if err != nil { + return nil, err + } + if IsCodexFormatAgent(agent) { + for j := range toolCalls { + for k := range toolCalls[j].ResultEvents { + db.PrepareToolResultEvent(&toolCalls[j].ResultEvents[k]) + } + } + } + toolResults, err := ConvertToolResultsContext(ctx, message.ToolResults) + if err != nil { + return nil, err + } + messages[i] = db.Message{ + SessionID: sessionID, Ordinal: message.Ordinal, + Role: string(message.Role), Content: message.Content, + ThinkingText: message.ThinkingText, + Timestamp: timeutil.Format(message.Timestamp), + HasThinking: message.HasThinking, HasToolUse: message.HasToolUse, + ContentLength: message.ContentLength, IsSystem: message.IsSystem, + Model: message.Model, ReasoningEffort: message.ReasoningEffort, + ProviderID: message.ProviderID, TokenUsage: message.TokenUsage, + ContextTokens: message.ContextTokens, OutputTokens: message.OutputTokens, + HasContextTokens: hasContext, HasOutputTokens: hasOutput, + ClaudeMessageID: message.ClaudeMessageID, + ClaudeRequestID: message.ClaudeRequestID, + SourceType: message.SourceType, SourceSubtype: message.SourceSubtype, + PromptSource: message.PromptSource, SourceUUID: message.SourceUUID, + SourceParentUUID: message.SourceParentUUID, + IsSidechain: message.IsSidechain, + IsCompactBoundary: message.IsCompactBoundary, + ToolCalls: toolCalls, ToolResults: toolResults, + } + } + return PairAndFilterContext(ctx, messages, blocked) +} + +// ConvertUsageEventsContext converts, stamps the final session ID, and +// validates parser usage rows. +func ConvertUsageEventsContext( + ctx context.Context, sessionID string, parsed []parser.ParsedUsageEvent, +) ([]db.UsageEvent, db.ValidationStats, error) { + events := make([]db.UsageEvent, 0, len(parsed)) + for _, event := range parsed { + if err := ctx.Err(); err != nil { + return nil, db.ValidationStats{}, err + } + events = append(events, db.UsageEvent{ + SessionID: sessionID, MessageOrdinal: event.MessageOrdinal, + Source: event.Source, Model: event.Model, ProviderID: event.ProviderID, + InputTokens: event.InputTokens, OutputTokens: event.OutputTokens, + CacheCreationInputTokens: event.CacheCreationInputTokens, + CacheReadInputTokens: event.CacheReadInputTokens, + ReasoningTokens: event.ReasoningTokens, Cost: event.Cost, + CostStatus: event.CostStatus, CostSource: event.CostSource, + OccurredAt: event.OccurredAt, DedupKey: event.DedupKey, + }) + } + stats, err := db.ValidateAndSanitizeContext(ctx, nil, nil, events) + return events, stats, err +} + +// ConvertToolCallsContext converts parsed tool calls into normalized rows. +func ConvertToolCallsContext( + ctx context.Context, sessionID string, parsed []parser.ParsedToolCall, +) ([]db.ToolCall, error) { + if len(parsed) == 0 { + return nil, ctx.Err() + } + calls := make([]db.ToolCall, len(parsed)) + for i, call := range parsed { + if err := ctx.Err(); err != nil { + return nil, err + } + filePath := call.FilePath + if filePath == "" { + filePath = parser.ResolveFilePathFromJSON(call.InputJSON) + } + resultEvents, err := ConvertToolResultEventsContext(ctx, call.ResultEvents) + if err != nil { + return nil, err + } + calls[i] = db.ToolCall{ + SessionID: sessionID, ToolName: call.ToolName, + Category: call.Category, ToolUseID: call.ToolUseID, + InputJSON: call.InputJSON, Rendering: call.Rendering, + FilePath: filePath, CallIndex: i, SkillName: call.SkillName, + SubagentSessionID: call.SubagentSessionID, + ResultEvents: resultEvents, + } + } + return calls, ctx.Err() +} + +// ConvertToolResultEventsContext converts structured result events. +func ConvertToolResultEventsContext( + ctx context.Context, parsed []parser.ParsedToolResultEvent, +) ([]db.ToolResultEvent, error) { + if len(parsed) == 0 { + return nil, ctx.Err() + } + events := make([]db.ToolResultEvent, len(parsed)) + for i, event := range parsed { + if err := ctx.Err(); err != nil { + return nil, err + } + events[i] = db.ToolResultEvent{ + RawContentDigest: event.RawContentDigest, ToolUseID: event.ToolUseID, + AgentID: event.AgentID, SubagentSessionID: event.SubagentSessionID, + Source: event.Source, Status: event.Status, Content: event.Content, + ContentLength: len(event.Content), Timestamp: timeutil.Format(event.Timestamp), + EventIndex: i, + } + } + return events, ctx.Err() +} + +// ConvertToolResultsContext converts transient result carriers for pairing. +func ConvertToolResultsContext( + ctx context.Context, parsed []parser.ParsedToolResult, +) ([]db.ToolResult, error) { + if len(parsed) == 0 { + return nil, ctx.Err() + } + results := make([]db.ToolResult, len(parsed)) + for i, result := range parsed { + if err := ctx.Err(); err != nil { + return nil, err + } + results[i] = db.ToolResult{ + ToolUseID: result.ToolUseID, ContentLength: result.ContentLength, + ContentRaw: result.ContentRaw, + } + } + return results, ctx.Err() +} + +// IsCodexFormatAgent reports agents using Codex rollout result identity rules. +func IsCodexFormatAgent(agent parser.AgentType) bool { + return agent == parser.AgentCodex || agent == parser.AgentTraeX +} + +// NextParsedOrdinal returns the resume ordinal after the final parsed row. +func NextParsedOrdinal(current int, messages []parser.ParsedMessage) int { + if len(messages) == 0 { + return current + } + return messages[len(messages)-1].Ordinal + 1 +} + +// LastParsedSourceUUID returns the last available source UUID for resumption. +func LastParsedSourceUUID( + current string, messages []parser.ParsedMessage, +) string { + for _, message := range slices.Backward(messages) { + if message.SourceUUID != "" { + return message.SourceUUID + } + } + return current +} + +func stringPtr(value string) *string { + if value == "" { + return nil + } + return &value +} + +func int64Ptr(value int64) *int64 { + if value == 0 { + return nil + } + return &value +} + +func cloneStringPtr(value *string) *string { + if value == nil { + return nil + } + clone := *value + return &clone +} diff --git a/internal/ingest/history.go b/internal/ingest/history.go new file mode 100644 index 0000000000..269daccbe0 --- /dev/null +++ b/internal/ingest/history.go @@ -0,0 +1,117 @@ +package ingest + +import ( + "context" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +// HistoryAction describes how a candidate relates to its same-source prior +// projection. +type HistoryAction uint8 + +const ( + HistoryReplace HistoryAction = iota + HistoryPreserve + HistoryMerge +) + +// PriorSession is the normalized state previously accepted from this exact +// source member. Callers own loading it and surfacing any read error. +type PriorSession struct { + Session db.Session + Messages []db.Message +} + +// HistoryResult contains the effective rows and the decision a storage caller +// must apply. PriorContributed tells provenance writers to retain the prior +// accepted source generation. +type HistoryResult struct { + Candidate Candidate + Action HistoryAction + PriorContributed bool +} + +// ReconcileProviderHistory applies provider-specific archive preservation to +// explicitly supplied current and same-source prior rows. It performs no +// storage or filesystem reads. +func ReconcileProviderHistory( + ctx context.Context, + candidate Candidate, + prior *PriorSession, + options ContentOptions, +) (HistoryResult, error) { + result := HistoryResult{Candidate: candidate, Action: HistoryReplace} + if err := ctx.Err(); err != nil || prior == nil { + return result, err + } + // Apply source-owned write intent before finalization and content hashing. + // Immutable projection rows cannot inherit these fields through an upsert. + if candidate.Session.PreserveSessionName { + candidate.Session.SessionName = prior.Session.SessionName + result.Candidate = candidate + result.PriorContributed = true + } + agent := candidate.Parsed.Session.Agent + if agent == "" { + agent = parser.AgentType(candidate.Session.Agent) + } + if IsOpenCodeFormatStorageAgent(agent) { + if shouldPreserveOpenCodeFormatArchive( + agent, candidate, *prior, options, + ) { + result.Candidate.Session = prior.Session + result.Candidate.Messages = prior.Messages + result.Action = HistoryPreserve + result.PriorContributed = true + } + return result, ctx.Err() + } + + switch agent { + case parser.AgentRooCode, parser.AgentKiloLegacy: + if len(candidate.Messages) == 0 && len(prior.Messages) > 0 { + result.Candidate.Session = prior.Session + result.Candidate.Messages = prior.Messages + result.Action = HistoryPreserve + result.PriorContributed = true + } + return result, ctx.Err() + case parser.AgentVSCopilot: + decision := visualStudioCopilotArchiveDecision( + candidate.Messages, prior.Messages, + ) + if decision.preserve { + decision.merged = prior.Messages + } + if decision.merged == nil { + return result, ctx.Err() + } + parsedMessages := candidate.Messages + merged, _ := db.ProjectToolResultImages( + decision.merged, options.ToolResultImages, + ) + candidate.Messages = merged + applyVisualStudioCopilotArchiveSessionFields( + &candidate.Session, &prior.Session, parsedMessages, merged, + ) + if err := ApplySessionMessageDerivedFieldsContext( + ctx, &candidate.Session, merged, + candidate.Parsed.Session.CountsAuthoritative, + ); err != nil { + return HistoryResult{}, err + } + if err := ApplySessionTokenTotalsFromMessagesContext( + ctx, &candidate.Session, merged, + ); err != nil { + return HistoryResult{}, err + } + result.Candidate = candidate + result.Action = HistoryMerge + result.PriorContributed = true + return result, ctx.Err() + default: + return result, ctx.Err() + } +} diff --git a/internal/ingest/history_opencode.go b/internal/ingest/history_opencode.go new file mode 100644 index 0000000000..8369831342 --- /dev/null +++ b/internal/ingest/history_opencode.go @@ -0,0 +1,304 @@ +package ingest + +import ( + "path/filepath" + "strings" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/usagefacts" +) + +func shouldPreserveOpenCodeFormatArchive( + agent parser.AgentType, + candidate Candidate, + prior PriorSession, + options ContentOptions, +) bool { + path := candidate.Parsed.Session.File.Path + if path == "" { + path = derefString(candidate.Session.FilePath) + } + if IsClaudeFormatTranscript(agent, path) { + return false + } + storedHash := derefString(prior.Session.FileHash) + storedPath := derefString(prior.Session.FilePath) + storedMtime := derefInt64(prior.Session.FileMtime) + currentMtime := candidate.Parsed.Session.File.Mtime + if currentMtime == 0 { + currentMtime = derefInt64(candidate.Session.FileMtime) + } + currentHash := candidate.Parsed.Session.File.Hash + if currentHash == "" { + currentHash = derefString(candidate.Session.FileHash) + } + storedHasStorageFingerprint := parser.HasOpenCodeStorageFingerprint(storedHash) + currentIsStorage := IsOpenCodeFormatStoragePath(agent, path) || + IsOpenCodeFormatSQLiteVirtualPath(agent, path) + storedIsStorage := IsOpenCodeFormatStoragePath(agent, storedPath) || + IsOpenCodeFormatSQLiteVirtualPath(agent, storedPath) || + (storedPath == "" && storedHasStorageFingerprint) + if !currentIsStorage && !storedIsStorage { + return false + } + storedIsSQLiteVirtual := IsOpenCodeFormatSQLiteVirtualPath(agent, storedPath) + storedIsStorageArchive := IsOpenCodeFormatStoragePath(agent, storedPath) || + (storedPath == "" && storedHasStorageFingerprint) + if storedIsSQLiteVirtual { + storedIsStorageArchive = false + } + if IsOpenCodeFormatSQLiteVirtualPath(agent, path) && !storedIsStorageArchive { + return false + } + if len(prior.Messages) == 0 { + return false + } + if storedHasStorageFingerprint && + parser.HasOpenCodeStorageFingerprint(currentHash) && + !parser.OpenCodeStorageFingerprintMissing(storedHash, currentHash) { + return false + } + if storedIsStorageArchive && + IsOpenCodeFormatSQLiteVirtualPath(agent, path) && + currentMtime != 0 && storedMtime != 0 && currentMtime <= storedMtime { + return true + } + if options.ArchiveContent.UsageOnly() { + _, currentMessages := db.ProjectSessionForStoragePolicy( + db.Session{}, candidate.Messages, options.ArchiveContent, + ) + _, storedMessages := db.ProjectSessionForStoragePolicy( + db.Session{}, prior.Messages, options.ArchiveContent, + ) + return OpenCodeUsageOnlyArchiveLooksIncomplete( + currentMessages, storedMessages, + ) + } + return OpenCodeLegacyArchiveLooksIncomplete( + candidate.Messages, prior.Messages, + ) +} + +// IsClaudeFormatTranscript reports whether path is a session-bearing Claude +// projects-layout transcript for an agent that uses the shared DAG parser. +func IsClaudeFormatTranscript(agent parser.AgentType, path string) bool { + if !IsClaudeFormatAgent(agent) { + return false + } + name := filepath.Base(path) + return name != ".jsonl" && strings.HasSuffix(name, ".jsonl") +} + +// IsClaudeFormatAgent reports agents that use the shared Claude DAG parser. +func IsClaudeFormatAgent(agent parser.AgentType) bool { + return agent == parser.AgentClaude || agent == parser.AgentIcodemate +} + +// IsOpenCodeFormatStorageAgent reports agents that consume OpenCode storage. +func IsOpenCodeFormatStorageAgent(agent parser.AgentType) bool { + return agent == parser.AgentOpenCode || + agent == parser.AgentKilo || + agent == parser.AgentIcodemate || + agent == parser.AgentMiMoCode +} + +// OpenCodeFormatDBName returns the provider's OpenCode-format container name. +func OpenCodeFormatDBName(agent parser.AgentType) string { + switch agent { + case parser.AgentOpenCode: + return "opencode.db" + case parser.AgentKilo: + return "kilo.db" + case parser.AgentMiMoCode: + return "mimocode.db" + case parser.AgentIcodemate: + return "icodemate.db" + default: + return "" + } +} + +// IsOpenCodeFormatStoragePath reports a JSON storage member path. +func IsOpenCodeFormatStoragePath(agent parser.AgentType, path string) bool { + return strings.HasSuffix(path, ".json") && + !IsOpenCodeFormatSQLiteVirtualPath(agent, path) +} + +// IsOpenCodeFormatContainerSource reports a storage member or SQLite member. +func IsOpenCodeFormatContainerSource(agent parser.AgentType, path string) bool { + return IsOpenCodeFormatStorageAgent(agent) && + (IsOpenCodeFormatStoragePath(agent, path) || + IsOpenCodeFormatSQLiteVirtualPath(agent, path)) +} + +// IsOpenCodeFormatSQLiteVirtualPath reports a provider container member path. +func IsOpenCodeFormatSQLiteVirtualPath( + agent parser.AgentType, path string, +) bool { + if !IsOpenCodeFormatStorageAgent(agent) { + return false + } + if agent == parser.AgentOpenCode { + _, _, ok := parser.ParseOpenCodeSQLiteVirtualPath(path) + return ok + } + _, _, ok := parser.ParseVirtualSourcePathForBase( + path, OpenCodeFormatDBName(agent), + ) + return ok +} + +// OpenCodeLegacyArchiveLooksIncomplete compares full-content rows using the +// historical OpenCode archive rules. +func OpenCodeLegacyArchiveLooksIncomplete(parsed, stored []db.Message) bool { + if parsed == nil { + return len(stored) > 0 + } + if len(parsed) < len(stored) { + return true + } + for i := range stored { + if openCodeMessageLooksIncomplete(parsed[i], stored[i]) { + return true + } + } + return false +} + +// OpenCodeUsageOnlyArchiveLooksIncomplete compares the fields retained by the +// usage-only storage policy, including stable message and delegation identity. +func OpenCodeUsageOnlyArchiveLooksIncomplete(parsed, stored []db.Message) bool { + if parsed == nil { + return len(stored) > 0 + } + if len(parsed) < len(stored) { + return true + } + parsedByIdentity := make(map[openCodeMessageIdentity]db.Message, len(parsed)*2) + for _, message := range parsed { + parsedByIdentity[openCodeMessageStorageIdentity(message)] = message + parsedByIdentity[openCodeMessageIdentity{ + ordinal: message.Ordinal, role: message.Role, + }] = message + } + for _, storedMessage := range stored { + identity := openCodeMessageStorageIdentity(storedMessage) + parsedMessage, ok := parsedByIdentity[identity] + if !ok || openCodeUsageOnlyMessageLooksIncomplete( + parsedMessage, storedMessage, + ) { + return true + } + } + return false +} + +type openCodeMessageIdentity struct { + sourceUUID string + ordinal int + role string +} + +func openCodeMessageStorageIdentity(message db.Message) openCodeMessageIdentity { + if message.SourceUUID != "" { + return openCodeMessageIdentity{sourceUUID: message.SourceUUID} + } + return openCodeMessageIdentity{ordinal: message.Ordinal, role: message.Role} +} + +func openCodeMessageLooksIncomplete(parsed, stored db.Message) bool { + if parsed.Ordinal != stored.Ordinal || parsed.Role != stored.Role { + return false + } + if sanitizedMessageContentLength(parsed) < sanitizedMessageContentLength(stored) { + return true + } + if parsed.HasThinking != stored.HasThinking && stored.HasThinking { + return true + } + if stored.HasOutputTokens && + (!parsed.HasOutputTokens || parsed.OutputTokens < stored.OutputTokens) { + return true + } + if stored.HasContextTokens && + (!parsed.HasContextTokens || parsed.ContextTokens < stored.ContextTokens) { + return true + } + if len(parsed.ToolCalls) < len(stored.ToolCalls) { + return true + } + return countToolResultEvents(parsed.ToolCalls) < + countToolResultEvents(stored.ToolCalls) +} + +func openCodeUsageOnlyMessageLooksIncomplete(parsed, stored db.Message) bool { + if parsed.Role != stored.Role { + return false + } + if openCodeUsageLooksIncomplete(parsed, stored) { + return true + } + parsedLinks := make(map[string]string, len(parsed.ToolCalls)) + for _, call := range parsed.ToolCalls { + parsedLinks[call.ToolUseID] = call.SubagentSessionID + } + for _, call := range stored.ToolCalls { + child, ok := parsedLinks[call.ToolUseID] + if !ok || child != call.SubagentSessionID { + return true + } + } + return false +} + +func openCodeUsageLooksIncomplete(parsed, stored db.Message) bool { + if stored.Model != "" && parsed.Model == "" { + return true + } + if len(stored.TokenUsage) == 0 { + return false + } + if len(parsed.TokenUsage) == 0 { + return true + } + parsedUsage := usagefacts.ParseTokenUsage(string(parsed.TokenUsage)) + storedUsage := usagefacts.ParseTokenUsage(string(stored.TokenUsage)) + return parsedUsage.InputTokens < storedUsage.InputTokens || + parsedUsage.OutputTokens < storedUsage.OutputTokens || + parsedUsage.ReasoningTokens < storedUsage.ReasoningTokens || + parsedUsage.CacheCreationTokens < storedUsage.CacheCreationTokens || + parsedUsage.CacheCreation1hTokens < storedUsage.CacheCreation1hTokens || + parsedUsage.CacheReadTokens < storedUsage.CacheReadTokens || + parsedUsage.WebSearchRequests < storedUsage.WebSearchRequests +} + +func sanitizedMessageContentLength(message db.Message) int { + sanitized := db.SanitizeUTF8(message.Content) + if sanitized != message.Content { + return len(sanitized) + } + return message.ContentLength +} + +func countToolResultEvents(calls []db.ToolCall) int { + total := 0 + for _, call := range calls { + total += len(call.ResultEvents) + } + return total +} + +func derefString(value *string) string { + if value == nil { + return "" + } + return *value +} + +func derefInt64(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} diff --git a/internal/ingest/history_test.go b/internal/ingest/history_test.go new file mode 100644 index 0000000000..3908951fb8 --- /dev/null +++ b/internal/ingest/history_test.go @@ -0,0 +1,249 @@ +package ingest_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" +) + +func TestReconcileProviderHistoryPreservesEmptyRooArchive(t *testing.T) { + prior := &ingest.PriorSession{ + Session: db.Session{ID: "roocode:task-1", Agent: string(parser.AgentRooCode)}, + Messages: []db.Message{{ + SessionID: "roocode:task-1", Ordinal: 0, Role: "user", + Content: "keep this", ContentLength: len("keep this"), + }}, + } + candidate := ingest.Candidate{ + Session: db.Session{ID: "roocode:task-1", Agent: string(parser.AgentRooCode)}, + Parsed: parser.ParseResult{Session: parser.ParsedSession{ + ID: "task-1", Agent: parser.AgentRooCode, + }}, + } + + result, err := ingest.ReconcileProviderHistory( + context.Background(), candidate, prior, ingest.ContentOptions{}, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryPreserve, result.Action) + assert.True(t, result.PriorContributed) + require.Len(t, result.Candidate.Messages, 1) + assert.Equal(t, "keep this", result.Candidate.Messages[0].Content) +} + +func TestReconcileProviderHistoryOpenCodeFingerprintAndContent(t *testing.T) { + const storedFingerprint = `opencode-storage:v1:{"messages":[{"id":"message-1","time":1,"hash":"old"}]}` + const currentFingerprint = `opencode-storage:v1:{"messages":[{"id":"message-1","time":1,"hash":"new"}]}` + storedPath := "/synthetic/storage/session/session-1.json" + prior := &ingest.PriorSession{ + Session: db.Session{ + ID: "opencode:session-1", Agent: string(parser.AgentOpenCode), + FilePath: &storedPath, FileHash: new(storedFingerprint), + }, + Messages: []db.Message{{ + Ordinal: 0, Role: "assistant", SourceUUID: "message-1", + Content: "complete answer", ContentLength: len("complete answer"), + }}, + } + + t.Run("missing fingerprint and weaker content preserves", func(t *testing.T) { + candidate := openCodeHistoryCandidate("", "short") + result, err := ingest.ReconcileProviderHistory( + context.Background(), candidate, prior, ingest.ContentOptions{}, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryPreserve, result.Action) + assert.True(t, result.PriorContributed) + assert.Equal(t, "complete answer", result.Candidate.Messages[0].Content) + }) + + t.Run("changed fingerprint with complete rewrite replaces", func(t *testing.T) { + candidate := openCodeHistoryCandidate(currentFingerprint, "a longer rewritten answer") + result, err := ingest.ReconcileProviderHistory( + context.Background(), candidate, prior, ingest.ContentOptions{}, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryReplace, result.Action) + assert.False(t, result.PriorContributed) + assert.Equal(t, "a longer rewritten answer", result.Candidate.Messages[0].Content) + }) +} + +func TestReconcileProviderHistoryOpenCodeUsageOnlyComparesUsage(t *testing.T) { + storedPath := "/synthetic/storage/session/session-1.json" + prior := &ingest.PriorSession{ + Session: db.Session{ + ID: "opencode:session-1", Agent: string(parser.AgentOpenCode), + FilePath: &storedPath, + }, + Messages: []db.Message{{ + Ordinal: 0, Role: "assistant", SourceUUID: "message-1", + Model: "model-a", + TokenUsage: []byte(`{"input_tokens":300,"output_tokens":200}`), + }}, + } + candidate := openCodeHistoryCandidate("", "content is projected away") + candidate.Messages[0].Model = "model-a" + candidate.Messages[0].TokenUsage = []byte( + `{"input_tokens":300,"output_tokens":199}`, + ) + + result, err := ingest.ReconcileProviderHistory( + context.Background(), candidate, prior, ingest.ContentOptions{ + ArchiveContent: config.ArchiveContentUsage, + }, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryPreserve, result.Action) + assert.True(t, result.PriorContributed) + assert.JSONEq(t, `{"input_tokens":300,"output_tokens":200}`, + string(result.Candidate.Messages[0].TokenUsage)) +} + +func TestReconcileProviderHistoryMergesVisualStudioCopilotArchive(t *testing.T) { + archivedFirst := "Archived prompt" + archivedName := "Archived title" + archivedStart := "2026-09-11T09:00:00Z" + archivedEnd := "2026-09-11T09:30:00Z" + prior := &ingest.PriorSession{ + Session: db.Session{ + ID: "copilot:conversation-1", Agent: string(parser.AgentVSCopilot), + FirstMessage: &archivedFirst, SessionName: &archivedName, + StartedAt: &archivedStart, EndedAt: &archivedEnd, + }, + Messages: []db.Message{ + { + SessionID: "copilot:conversation-1", Ordinal: 0, Role: "user", + Content: archivedFirst, ContentLength: len(archivedFirst), + Timestamp: archivedStart, + }, + { + SessionID: "copilot:conversation-1", Ordinal: 1, Role: "assistant", + Content: "running", ContentLength: len("running"), + Timestamp: "2026-09-11T09:10:00Z", + ToolCalls: []db.ToolCall{{ + ToolUseID: "call-1", ToolName: "run_command_in_terminal", + }}, + }, + }, + } + candidate := ingest.Candidate{ + Session: db.Session{ + ID: "copilot:conversation-1", Agent: string(parser.AgentVSCopilot), + StartedAt: new("2026-09-11T09:10:00Z"), + EndedAt: new("2026-09-11T09:20:00Z"), + }, + Messages: []db.Message{ + { + SessionID: "copilot:conversation-1", Ordinal: 0, Role: "assistant", + Content: "running", ContentLength: len("running"), + Timestamp: "2026-09-11T09:10:00Z", OutputTokens: 10, + HasOutputTokens: true, + ToolCalls: []db.ToolCall{{ + ToolUseID: "call-1", ToolName: "run_command_in_terminal", + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "call-1", Status: "completed", + Content: "passed", ContentLength: len("passed"), + }}, + }}, + }, + { + SessionID: "copilot:conversation-1", Ordinal: 1, Role: "assistant", + Content: "new answer", ContentLength: len("new answer"), + Timestamp: "2026-09-11T09:20:00Z", OutputTokens: 5, + HasOutputTokens: true, + }, + }, + Parsed: parser.ParseResult{Session: parser.ParsedSession{ + ID: "conversation-1", Agent: parser.AgentVSCopilot, + }}, + } + + result, err := ingest.ReconcileProviderHistory( + context.Background(), candidate, prior, ingest.ContentOptions{}, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryMerge, result.Action) + assert.True(t, result.PriorContributed) + require.Len(t, result.Candidate.Messages, 3) + assert.Equal(t, archivedFirst, result.Candidate.Messages[0].Content) + require.Len(t, result.Candidate.Messages[1].ToolCalls, 1) + require.Len(t, result.Candidate.Messages[1].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, "passed", + result.Candidate.Messages[1].ToolCalls[0].ResultEvents[0].Content) + assert.Equal(t, "new answer", result.Candidate.Messages[2].Content) + assert.Equal(t, &archivedFirst, result.Candidate.Session.FirstMessage) + assert.Equal(t, &archivedName, result.Candidate.Session.SessionName) + assert.Equal(t, &archivedStart, result.Candidate.Session.StartedAt) + assert.Equal(t, &archivedEnd, result.Candidate.Session.EndedAt) + assert.Equal(t, 3, result.Candidate.Session.MessageCount) + assert.Equal(t, 1, result.Candidate.Session.UserMessageCount) + assert.Equal(t, 15, result.Candidate.Session.TotalOutputTokens) + assert.True(t, result.Candidate.Session.HasTotalOutputTokens) +} + +func TestProviderHistorySourceClassifiers(t *testing.T) { + tests := []struct { + name string + agent parser.AgentType + path string + claude bool + storage bool + sqliteVirtual bool + }{ + { + name: "claude transcript", agent: parser.AgentClaude, + path: "/synthetic/project/session-1.jsonl", claude: true, + }, + { + name: "icodemate cli transcript", agent: parser.AgentIcodemate, + path: "/synthetic/project/session-1.jsonl", claude: true, + }, + { + name: "icodemate storage json", agent: parser.AgentIcodemate, + path: "/synthetic/storage/session/session-1.json", storage: true, + }, + { + name: "opencode sqlite member", agent: parser.AgentOpenCode, + path: "/synthetic/opencode.db#session-1", sqliteVirtual: true, + }, + { + name: "kilo sqlite member", agent: parser.AgentKilo, + path: "/synthetic/kilo.db#session-1", sqliteVirtual: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.claude, + ingest.IsClaudeFormatTranscript(test.agent, test.path)) + assert.Equal(t, test.storage, + ingest.IsOpenCodeFormatStoragePath(test.agent, test.path)) + assert.Equal(t, test.sqliteVirtual, + ingest.IsOpenCodeFormatSQLiteVirtualPath(test.agent, test.path)) + }) + } +} + +func openCodeHistoryCandidate(hash, content string) ingest.Candidate { + return ingest.Candidate{ + Session: db.Session{ + ID: "opencode:session-1", Agent: string(parser.AgentOpenCode), + }, + Messages: []db.Message{{ + Ordinal: 0, Role: "assistant", SourceUUID: "message-1", + Content: content, ContentLength: len(content), + }}, + Parsed: parser.ParseResult{Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentOpenCode, + File: parser.FileInfo{ + Path: "/synthetic/storage/session/session-1.json", Hash: hash, + }, + }}, + } +} diff --git a/internal/ingest/history_vscopilot.go b/internal/ingest/history_vscopilot.go new file mode 100644 index 0000000000..0302358ea1 --- /dev/null +++ b/internal/ingest/history_vscopilot.go @@ -0,0 +1,438 @@ +package ingest + +import ( + "fmt" + "slices" + "strings" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +type visualStudioCopilotArchiveReconcile struct { + preserve bool + merged []db.Message +} + +// ReconcileVisualStudioCopilotArchive exposes the pure message decision for +// compatibility callers that do not need session-field reconciliation. +func ReconcileVisualStudioCopilotArchive( + parsed, stored []db.Message, +) (merged []db.Message, preserve bool) { + decision := visualStudioCopilotArchiveDecision(parsed, stored) + return decision.merged, decision.preserve +} + +func visualStudioCopilotArchiveDecision( + parsed, stored []db.Message, +) visualStudioCopilotArchiveReconcile { + if len(stored) == 0 { + return visualStudioCopilotArchiveReconcile{} + } + if parsed == nil { + return visualStudioCopilotArchiveReconcile{preserve: true} + } + storedByKey := make(map[string][]int, len(stored)) + for i, message := range stored { + key := visualStudioCopilotMessagePresenceKey(message) + storedByKey[key] = append(storedByKey[key], i) + } + matchedStored := make([]bool, len(stored)) + updates := make(map[int]db.Message) + additions := make([]db.Message, 0) + hasIncomplete := false + for _, parsedMessage := range parsed { + key := visualStudioCopilotMessagePresenceKey(parsedMessage) + candidates := storedByKey[key] + if len(candidates) == 0 { + additions = append(additions, parsedMessage) + continue + } + storedIndex := candidates[0] + storedByKey[key] = candidates[1:] + matchedStored[storedIndex] = true + storedMessage := stored[storedIndex] + incomplete := VisualStudioCopilotMessageLooksIncomplete( + parsedMessage, storedMessage, + ) + if incomplete { + hasIncomplete = true + } + if !incomplete && VisualStudioCopilotMessageHasArchiveUpdate( + parsedMessage, storedMessage, + ) { + updates[storedIndex] = parsedMessage + } + } + var fallbackMatched bool + additions, fallbackMatched = visualStudioCopilotResolveArchiveAdditions( + stored, matchedStored, updates, additions, &hasIncomplete, + ) + hasArchiveOnly := slices.Contains(matchedStored, false) + if hasIncomplete || hasArchiveOnly || fallbackMatched { + if len(updates) > 0 || len(additions) > 0 || + (fallbackMatched && !hasIncomplete) { + return visualStudioCopilotArchiveReconcile{merged: visualStudioCopilotMergeArchiveMessages( + stored, updates, additions, + )} + } + return visualStudioCopilotArchiveReconcile{preserve: true} + } + return visualStudioCopilotArchiveReconcile{} +} + +func visualStudioCopilotResolveArchiveAdditions( + stored []db.Message, + matchedStored []bool, + updates map[int]db.Message, + additions []db.Message, + hasIncomplete *bool, +) ([]db.Message, bool) { + matched := false + unresolved := additions[:0] + for _, parsedMessage := range additions { + storedIndex, ok := visualStudioCopilotArchiveFallbackMatch( + parsedMessage, stored, matchedStored, + ) + if !ok { + unresolved = append(unresolved, parsedMessage) + continue + } + matched = true + matchedStored[storedIndex] = true + storedMessage := stored[storedIndex] + incomplete := VisualStudioCopilotMessageLooksIncomplete( + parsedMessage, storedMessage, + ) + if incomplete { + *hasIncomplete = true + continue + } + update := visualStudioCopilotArchiveFallbackUpdate( + parsedMessage, storedMessage, + ) + if VisualStudioCopilotMessageHasArchiveUpdate(update, storedMessage) { + updates[storedIndex] = update + } + } + return unresolved, matched +} + +func visualStudioCopilotArchiveFallbackMatch( + parsed db.Message, stored []db.Message, matchedStored []bool, +) (int, bool) { + match := -1 + for i, storedMessage := range stored { + if matchedStored[i] || + !visualStudioCopilotMessagesFallbackMatch(parsed, storedMessage) { + continue + } + if match != -1 { + return 0, false + } + match = i + } + if match == -1 { + return 0, false + } + return match, true +} + +func visualStudioCopilotMessagesFallbackMatch(parsed, stored db.Message) bool { + if parsed.Role != stored.Role { + return false + } + if visualStudioCopilotMessagesShareToolIdentity(parsed, stored) { + return true + } + return visualStudioCopilotMessagesShareContentIdentity(parsed, stored) +} + +func visualStudioCopilotMessagesShareToolIdentity(parsed, stored db.Message) bool { + if len(parsed.ToolCalls) == 0 || len(stored.ToolCalls) == 0 { + return false + } + parsedIDs := make(map[string]string, len(parsed.ToolCalls)) + for _, call := range parsed.ToolCalls { + id := strings.TrimSpace(call.ToolUseID) + if id != "" { + parsedIDs[id] = strings.TrimSpace(call.ToolName) + } + } + for _, call := range stored.ToolCalls { + id := strings.TrimSpace(call.ToolUseID) + if id == "" { + continue + } + parsedName, ok := parsedIDs[id] + if !ok { + continue + } + storedName := strings.TrimSpace(call.ToolName) + if parsedName != "" && storedName != "" && parsedName != storedName { + continue + } + return true + } + return false +} + +func visualStudioCopilotMessagesShareContentIdentity(parsed, stored db.Message) bool { + if len(parsed.ToolCalls) > 0 || len(stored.ToolCalls) > 0 { + return false + } + switch parsed.Role { + case string(parser.RoleAssistant), string(parser.RoleUser): + default: + return false + } + return parsed.Content != "" && parsed.Content == stored.Content +} + +func visualStudioCopilotArchiveFallbackUpdate(parsed, stored db.Message) db.Message { + update := parsed + update.Timestamp = stored.Timestamp + return update +} + +func visualStudioCopilotMergeArchiveMessages( + stored []db.Message, updates map[int]db.Message, additions []db.Message, +) []db.Message { + merged := make([]db.Message, 0, len(stored)+len(additions)) + merged = append(merged, stored...) + for index, message := range updates { + merged[index] = message + } + merged = append(merged, additions...) + if len(additions) > 0 { + slices.SortStableFunc(merged, compareVisualStudioCopilotMessageOrder) + } + for i := range merged { + merged[i].Ordinal = i + } + return merged +} + +func compareVisualStudioCopilotMessageOrder(a, b db.Message) int { + aTime, aOK := visualStudioCopilotMessageTime(a) + bTime, bOK := visualStudioCopilotMessageTime(b) + if aOK && bOK { + switch { + case aTime.Before(bTime): + return -1 + case aTime.After(bTime): + return 1 + default: + return 0 + } + } + if aOK { + return -1 + } + if bOK { + return 1 + } + return 0 +} + +func visualStudioCopilotMessageTime(message db.Message) (time.Time, bool) { + if message.Timestamp == "" { + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339Nano, message.Timestamp) + return parsed, err == nil +} + +func visualStudioCopilotMessagePresenceKey(message db.Message) string { + if message.Timestamp != "" { + return message.Role + "\x00time\x00" + message.Timestamp + } + if message.SourceUUID != "" { + return message.Role + "\x00source\x00" + message.SourceUUID + } + return fmt.Sprintf("%s\x00ordinal\x00%d", message.Role, message.Ordinal) +} + +// VisualStudioCopilotMessageLooksIncomplete compares a parsed row with its +// stored counterpart after applying the storage sanitizer. +func VisualStudioCopilotMessageLooksIncomplete(parsed, stored db.Message) bool { + if parsed.Role != stored.Role { + return false + } + p := sanitizedForArchiveCompare(parsed) + if p.ContentLength < stored.ContentLength || + (stored.HasThinking && !p.HasThinking) || + (stored.HasOutputTokens && + (!p.HasOutputTokens || p.OutputTokens < stored.OutputTokens)) || + (stored.HasContextTokens && + (!p.HasContextTokens || p.ContextTokens < stored.ContextTokens)) || + len(p.ToolCalls) < len(stored.ToolCalls) || + countToolResultEvents(p.ToolCalls) < countToolResultEvents(stored.ToolCalls) { + return true + } + return countToolResultContentLength(p.ToolCalls) < + countToolResultContentLength(stored.ToolCalls) +} + +func sanitizedForArchiveCompare(message db.Message) db.Message { + db.SanitizeMessage(&message) + return message +} + +// VisualStudioCopilotMessageHasArchiveUpdate reports whether a complete parsed +// row adds richer content than its stored counterpart. +func VisualStudioCopilotMessageHasArchiveUpdate(parsed, stored db.Message) bool { + if parsed.Role != stored.Role { + return false + } + p := sanitizedForArchiveCompare(parsed) + if p.ContentLength > stored.ContentLength || + (p.ContentLength == stored.ContentLength && p.Content != stored.Content) || + (p.HasThinking && (!stored.HasThinking || p.ThinkingText != stored.ThinkingText)) || + (p.HasOutputTokens && + (!stored.HasOutputTokens || p.OutputTokens > stored.OutputTokens)) || + (p.HasContextTokens && + (!stored.HasContextTokens || p.ContextTokens > stored.ContextTokens)) || + (string(p.TokenUsage) != "" && string(p.TokenUsage) != string(stored.TokenUsage)) { + return true + } + return visualStudioCopilotToolCallsHaveArchiveUpdate(p.ToolCalls, stored.ToolCalls) +} + +func visualStudioCopilotToolCallsHaveArchiveUpdate(parsed, stored []db.ToolCall) bool { + if len(parsed) > len(stored) { + return true + } + for i := 0; i < len(parsed) && i < len(stored); i++ { + if visualStudioCopilotToolCallHasArchiveUpdate(parsed[i], stored[i]) { + return true + } + } + return false +} + +func visualStudioCopilotToolCallHasArchiveUpdate(parsed, stored db.ToolCall) bool { + if parsed.ResultContentLength > stored.ResultContentLength || + (parsed.ResultContentLength == stored.ResultContentLength && + parsed.ResultContent != "" && parsed.ResultContent != stored.ResultContent) || + len(parsed.ResultEvents) > len(stored.ResultEvents) { + return true + } + for i := 0; i < len(parsed.ResultEvents) && i < len(stored.ResultEvents); i++ { + parsedEvent := parsed.ResultEvents[i] + storedEvent := stored.ResultEvents[i] + if parsedEvent.ContentLength > storedEvent.ContentLength || + (parsedEvent.ContentLength == storedEvent.ContentLength && + parsedEvent.Content != "" && parsedEvent.Content != storedEvent.Content) || + (parsedEvent.Status != "" && parsedEvent.Status != storedEvent.Status) { + return true + } + } + return false +} + +func countToolResultContentLength(calls []db.ToolCall) int { + total := 0 + for _, call := range calls { + total += call.ResultContentLength + for _, event := range call.ResultEvents { + total += event.ContentLength + } + } + return total +} + +func applyVisualStudioCopilotArchiveSessionFields( + session *db.Session, + archived *db.Session, + parsedMessages, mergedMessages []db.Message, +) { + if archived == nil { + return + } + archiveExtendsBounds := sessionTimeBefore(archived.StartedAt, session.StartedAt) || + sessionTimeAfter(archived.EndedAt, session.EndedAt) + if !visualStudioCopilotMergedFirstMessageFromParsed( + parsedMessages, mergedMessages, + ) { + session.FirstMessage = cloneStringPtr(archived.FirstMessage) + } + if archiveExtendsBounds || stringPtrEmpty(session.SessionName) { + session.SessionName = cloneStringPtr(archived.SessionName) + } + session.StartedAt = earlierSessionTime(archived.StartedAt, session.StartedAt) + session.EndedAt = laterSessionTime(archived.EndedAt, session.EndedAt) +} + +func visualStudioCopilotMergedFirstMessageFromParsed( + parsed, merged []db.Message, +) bool { + if len(parsed) == 0 || len(merged) == 0 { + return false + } + mergedFirst := merged[0] + for _, parsedMessage := range parsed { + if visualStudioCopilotMessagePresenceKey(parsedMessage) != + visualStudioCopilotMessagePresenceKey(mergedFirst) { + continue + } + return !VisualStudioCopilotMessageLooksIncomplete(parsedMessage, mergedFirst) && + !VisualStudioCopilotMessageHasArchiveUpdate(mergedFirst, parsedMessage) + } + return false +} + +func stringPtrEmpty(value *string) bool { + return value == nil || strings.TrimSpace(*value) == "" +} + +func sessionTimeBefore(a, b *string) bool { + return sessionTimeCompares(a, b, time.Time.Before) +} + +func sessionTimeAfter(a, b *string) bool { + return sessionTimeCompares(a, b, time.Time.After) +} + +func sessionTimeCompares( + a, b *string, compare func(time.Time, time.Time) bool, +) bool { + if a == nil || b == nil { + return a != nil && b == nil + } + aTime, aErr := time.Parse(time.RFC3339Nano, *a) + bTime, bErr := time.Parse(time.RFC3339Nano, *b) + return aErr == nil && bErr == nil && compare(aTime, bTime) +} + +func earlierSessionTime(a, b *string) *string { + return chooseSessionTime(a, b, time.Time.Before) +} + +func laterSessionTime(a, b *string) *string { + return chooseSessionTime(a, b, time.Time.After) +} + +func chooseSessionTime( + a, b *string, chooseA func(time.Time, time.Time) bool, +) *string { + switch { + case a == nil: + return cloneStringPtr(b) + case b == nil: + return cloneStringPtr(a) + } + aTime, aErr := time.Parse(time.RFC3339Nano, *a) + bTime, bErr := time.Parse(time.RFC3339Nano, *b) + switch { + case aErr != nil: + return cloneStringPtr(b) + case bErr != nil: + return cloneStringPtr(a) + case chooseA(aTime, bTime): + return cloneStringPtr(a) + default: + return cloneStringPtr(b) + } +} diff --git a/internal/ingest/prepare.go b/internal/ingest/prepare.go new file mode 100644 index 0000000000..cf2aa5c4a1 --- /dev/null +++ b/internal/ingest/prepare.go @@ -0,0 +1,269 @@ +// Package ingest converts parser output into normalized database rows without +// requiring a storage handle. +package ingest + +import ( + "context" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +// ContentOptions defines the content policies applied during preparation. +type ContentOptions struct { + BlockedResultCategories map[string]bool + ToolResultImages config.ToolResultImages + ArchiveContent config.ArchiveContent +} + +// Candidate is normalized parser output before caller-owned identity, +// attribution, filtering, and provider-history reconciliation. +type Candidate struct { + Session db.Session + Messages []db.Message + Parsed parser.ParseResult +} + +// PreparedSession is the complete normalized content graph ready for storage. +type PreparedSession struct { + Session db.Session + Messages []db.Message + UsageEvents []db.UsageEvent + Signals db.SessionSignalUpdate + Findings []db.SecretFinding + Validation db.ValidationStats +} + +// PrepareCandidate converts and pairs parser output before caller-owned +// reconciliation changes the final identity or retained transcript. +func PrepareCandidate( + ctx context.Context, parsed parser.ParseResult, options ContentOptions, +) (Candidate, error) { + messages, err := ConvertMessagesContext( + ctx, parsed.Session.ID, parsed.Session.Agent, parsed.Messages, + options.BlockedResultCategories, + ) + if err != nil { + return Candidate{}, err + } + messages, _ = db.ProjectToolResultImages(messages, options.ToolResultImages) + session, err := ConvertSessionContext(ctx, parsed.Session, parsed.Messages) + if err != nil { + return Candidate{}, err + } + if err := ApplySessionMessageDerivedFieldsContext( + ctx, &session, messages, parsed.Session.CountsAuthoritative, + ); err != nil { + return Candidate{}, err + } + return Candidate{ + Session: session, Messages: messages, Parsed: parsed, + }, ctx.Err() +} + +// Finalize applies post-reconciliation validation, aggregate repair, storage +// projection, usage conversion, and content-derived state. +func Finalize( + ctx context.Context, candidate Candidate, options ContentOptions, +) (PreparedSession, error) { + return finalize(ctx, candidate, options, true) +} + +// FinalizeRows performs final row normalization without derived signals or +// findings. Local staged and explicitly disabled signal paths use it when a +// later transaction-owned computation is required or derivation is disabled. +func FinalizeRows( + ctx context.Context, candidate Candidate, options ContentOptions, +) (PreparedSession, error) { + return finalize(ctx, candidate, options, false) +} + +func finalize( + ctx context.Context, candidate Candidate, options ContentOptions, + derive bool, +) (PreparedSession, error) { + session := candidate.Session + messages := candidate.Messages + if err := ApplySessionMessageDerivedFieldsContext( + ctx, &session, messages, + candidate.Parsed.Session.CountsAuthoritative, + ); err != nil { + return PreparedSession{}, err + } + + msgTotal, msgHasOut, msgPeak, msgHasCtx, err := + MessageTokenTotalsContext(ctx, messages) + if err != nil { + return PreparedSession{}, err + } + evtTotal, evtHasOut, evtPeak, evtHasCtx, err := + UsageEventTokenTotalsContext(ctx, candidate.Parsed.UsageEvents, false) + if err != nil { + return PreparedSession{}, err + } + totalFromMessages := session.HasTotalOutputTokens == msgHasOut && + session.TotalOutputTokens == msgTotal + totalFromEvents := session.HasTotalOutputTokens == evtHasOut && + session.TotalOutputTokens == evtTotal + peakFromMessages := session.HasPeakContextTokens == msgHasCtx && + session.PeakContextTokens == msgPeak + peakFromEvents := session.HasPeakContextTokens == evtHasCtx && + session.PeakContextTokens == evtPeak + + validation, err := db.ValidateAndSanitizeContext( + ctx, &session, messages, nil, + ) + if err != nil { + return PreparedSession{}, err + } + if totalFromMessages { + session.TotalOutputTokens, session.HasTotalOutputTokens, + _, _, err = MessageTokenTotalsContext(ctx, messages) + } else if totalFromEvents { + session.TotalOutputTokens, session.HasTotalOutputTokens, + _, _, err = UsageEventTokenTotalsContext( + ctx, candidate.Parsed.UsageEvents, true, + ) + } + if err != nil { + return PreparedSession{}, err + } + if peakFromMessages { + _, _, session.PeakContextTokens, session.HasPeakContextTokens, + err = MessageTokenTotalsContext(ctx, messages) + } else if peakFromEvents { + _, _, session.PeakContextTokens, session.HasPeakContextTokens, + err = UsageEventTokenTotalsContext( + ctx, candidate.Parsed.UsageEvents, true, + ) + } + if err != nil { + return PreparedSession{}, err + } + + usageEvents, usageValidation, err := ConvertUsageEventsContext( + ctx, session.ID, candidate.Parsed.UsageEvents, + ) + if err != nil { + return PreparedSession{}, err + } + validation = addValidationStats(validation, usageValidation) + session, messages = db.ProjectSessionForStoragePolicy( + session, messages, options.ArchiveContent, + ) + var signalUpdate db.SessionSignalUpdate + var findings []db.SecretFinding + if derive { + signalUpdate, findings = ComputeSignalsAndSecrets(session, messages) + } + return PreparedSession{ + Session: session, Messages: messages, UsageEvents: usageEvents, + Signals: signalUpdate, Findings: findings, Validation: validation, + }, ctx.Err() +} + +// ApplySessionMessageDerivedFieldsContext derives filtered counts and +// automation from the reconciled transcript. +func ApplySessionMessageDerivedFieldsContext( + ctx context.Context, session *db.Session, messages []db.Message, + countsAuthoritative bool, +) error { + if !countsAuthoritative { + var user int + for _, message := range messages { + if err := ctx.Err(); err != nil { + return err + } + if message.Role == "user" && !message.IsSystem && + message.SourceSubtype != parser.SourceSubtypeToolResult { + user++ + } + } + session.MessageCount = len(messages) + session.UserMessageCount = user + } + session.IsAutomated = db.IsAutomatedSessionMetadata( + session.Agent, session.SessionKind, + ) || db.IsAutomatedTranscript( + session.UserMessageCount, messages, session.FirstMessage, + ) + return ctx.Err() +} + +// MessageTokenTotalsContext returns row-derived output and context totals. +func MessageTokenTotalsContext( + ctx context.Context, messages []db.Message, +) (totalOutput int, hasOutput bool, peakContext int, hasContext bool, err error) { + for _, message := range messages { + if err = ctx.Err(); err != nil { + return + } + if message.HasOutputTokens { + hasOutput = true + totalOutput += message.OutputTokens + } + if message.HasContextTokens { + hasContext = true + if message.ContextTokens > peakContext { + peakContext = message.ContextTokens + } + } + } + err = ctx.Err() + return +} + +// ApplySessionTokenTotalsFromMessagesContext replaces session aggregates with +// totals derived from the supplied normalized message rows. +func ApplySessionTokenTotalsFromMessagesContext( + ctx context.Context, session *db.Session, messages []db.Message, +) error { + totalOutput, hasOutput, peakContext, hasContext, err := + MessageTokenTotalsContext(ctx, messages) + if err != nil { + return err + } + session.TotalOutputTokens = totalOutput + session.HasTotalOutputTokens = hasOutput + session.PeakContextTokens = peakContext + session.HasPeakContextTokens = hasContext + return nil +} + +// UsageEventTokenTotalsContext returns per-turn usage-derived totals, excluding +// authoritative session summaries. +func UsageEventTokenTotalsContext( + ctx context.Context, events []parser.ParsedUsageEvent, clamp bool, +) (totalOutput int, hasOutput bool, peakContext int, hasContext bool, err error) { + rolled := make([]parser.ParsedUsageEvent, 0, len(events)) + for _, event := range events { + if err = ctx.Err(); err != nil { + return + } + if event.Source == "session" { + continue + } + if clamp { + event.InputTokens = db.ClampParsedTokens(event.InputTokens) + event.OutputTokens = db.ClampParsedTokens(event.OutputTokens) + event.CacheCreationInputTokens = db.ClampParsedTokens( + event.CacheCreationInputTokens, + ) + event.CacheReadInputTokens = db.ClampParsedTokens( + event.CacheReadInputTokens, + ) + } + rolled = append(rolled, event) + } + return parser.UsageEventTokenAggregateContext(ctx, rolled) +} + +func addValidationStats(a, b db.ValidationStats) db.ValidationStats { + a.ControlCharsStripped += b.ControlCharsStripped + a.ModelClamped += b.ModelClamped + a.TokensClamped += b.TokensClamped + a.RoleCoerced += b.RoleCoerced + a.TimestampsBlanked += b.TimestampsBlanked + return a +} diff --git a/internal/ingest/prepare_test.go b/internal/ingest/prepare_test.go new file mode 100644 index 0000000000..23dc2dd63c --- /dev/null +++ b/internal/ingest/prepare_test.go @@ -0,0 +1,183 @@ +package ingest_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" +) + +func TestPrepareCandidatePairsToolResultsAndFiltersCarrier(t *testing.T) { + parsed := parser.ParseResult{ + Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentClaude, + }, + Messages: []parser.ParsedMessage{ + { + Ordinal: 2, Role: parser.RoleAssistant, Content: "checking", + ToolCalls: []parser.ParsedToolCall{{ + ToolUseID: "call-1", ToolName: "Bash", Category: "Bash", + }}, + }, + { + Ordinal: 3, Role: parser.RoleUser, + ToolResults: []parser.ParsedToolResult{{ + ToolUseID: "call-1", ContentLength: 99, + ContentRaw: `"command output"`, + }}, + }, + }, + } + + candidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + require.Len(t, candidate.Messages, 1) + require.Len(t, candidate.Messages[0].ToolCalls, 1) + assert.Equal(t, 2, candidate.Messages[0].Ordinal, + "filtering a result carrier must preserve parsed ordinals") + assert.Equal(t, "command output", + candidate.Messages[0].ToolCalls[0].ResultContent) + assert.Equal(t, len("command output"), + candidate.Messages[0].ToolCalls[0].ResultContentLength) + assert.Equal(t, 1, candidate.Session.MessageCount) +} + +func TestFinalizeClampsRowDerivedTokens(t *testing.T) { + parsed := parser.ParseResult{ + Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentVSCodeCopilot, + TotalOutputTokens: 999_999_999, + HasTotalOutputTokens: true, + }, + UsageEvents: []parser.ParsedUsageEvent{{ + Source: "turn", Model: "model-1", + InputTokens: -7, OutputTokens: 999_999_999, + }}, + } + candidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + + prepared, err := ingest.Finalize( + context.Background(), candidate, ingest.ContentOptions{}, + ) + require.NoError(t, err) + require.Len(t, prepared.UsageEvents, 1) + assert.Zero(t, prepared.UsageEvents[0].InputTokens) + assert.Equal(t, db.MaxPlausibleTokens, + prepared.UsageEvents[0].OutputTokens) + assert.Equal(t, db.MaxPlausibleTokens, + prepared.Session.TotalOutputTokens, + "row-derived total must follow the clamped row") + assert.Equal(t, 2, prepared.Validation.TokensClamped) +} + +func TestFinalizePreservesAuthoritativeSummaryTotals(t *testing.T) { + const summaryTotal = 4_242_424 + const summaryPeak = 3_333_333 + parsed := parser.ParseResult{ + Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentHermes, + TotalOutputTokens: summaryTotal, + HasTotalOutputTokens: true, + PeakContextTokens: summaryPeak, + HasPeakContextTokens: true, + }, + UsageEvents: []parser.ParsedUsageEvent{{ + Source: "session", Model: "model-1", + InputTokens: -11, OutputTokens: 17, + }}, + } + candidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + + prepared, err := ingest.Finalize( + context.Background(), candidate, ingest.ContentOptions{}, + ) + require.NoError(t, err) + require.Len(t, prepared.UsageEvents, 1) + assert.Zero(t, prepared.UsageEvents[0].InputTokens) + assert.Equal(t, 17, prepared.UsageEvents[0].OutputTokens) + assert.Equal(t, summaryTotal, prepared.Session.TotalOutputTokens) + assert.Equal(t, summaryPeak, prepared.Session.PeakContextTokens) +} + +func TestFinalizeKeepsUsageWithoutMessagesAndStampsFinalID(t *testing.T) { + parsed := parser.ParseResult{ + Session: parser.ParsedSession{ + ID: "native-id", Agent: parser.AgentHermes, + CountsAuthoritative: true, + }, + UsageEvents: []parser.ParsedUsageEvent{{ + SessionID: "native-id", Source: "session", Model: "model-1", + OutputTokens: 23, + }}, + } + candidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + candidate.Session.ID = "tenant~native-id" + + prepared, err := ingest.Finalize( + context.Background(), candidate, ingest.ContentOptions{}, + ) + require.NoError(t, err) + assert.Empty(t, prepared.Messages) + require.Len(t, prepared.UsageEvents, 1) + assert.Equal(t, "tenant~native-id", prepared.UsageEvents[0].SessionID) + assert.Equal(t, 23, prepared.UsageEvents[0].OutputTokens) +} + +func TestFinalizeProjectsStoredContentBeforeSecretFindings(t *testing.T) { + parsed := parser.ParseResult{ + Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentClaude, + }, + Messages: []parser.ParsedMessage{{ + Ordinal: 0, Role: parser.RoleAssistant, Content: "checking", + Timestamp: time.Date(2026, 9, 11, 10, 0, 0, 0, time.UTC), + ToolCalls: []parser.ParsedToolCall{{ + ToolUseID: "call-1", ToolName: "Bash", Category: "Bash", + InputJSON: `{"token":"AKIA7QHWN2DKR4FYPLJM"}`, + }}, + }}, + } + + fullCandidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + full, err := ingest.Finalize( + context.Background(), fullCandidate, ingest.ContentOptions{}, + ) + require.NoError(t, err) + require.NotEmpty(t, full.Findings, + "the full-content control must prove the fixture is detectable") + + projectedCandidate, err := ingest.PrepareCandidate( + context.Background(), parsed, ingest.ContentOptions{}, + ) + require.NoError(t, err) + projected, err := ingest.Finalize( + context.Background(), projectedCandidate, ingest.ContentOptions{ + ArchiveContent: config.ArchiveContentTranscripts, + }) + require.NoError(t, err) + require.Len(t, projected.Messages, 1) + require.Len(t, projected.Messages[0].ToolCalls, 1) + assert.Empty(t, projected.Messages[0].ToolCalls[0].InputJSON) + assert.Empty(t, projected.Findings, + "findings must describe only content retained by storage policy") +} diff --git a/internal/ingest/signals.go b/internal/ingest/signals.go new file mode 100644 index 0000000000..1016d7518e --- /dev/null +++ b/internal/ingest/signals.go @@ -0,0 +1,353 @@ +package ingest + +import ( + "slices" + "sync/atomic" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/secrets" + "go.kenn.io/agentsview/internal/signals" +) + +var secretScanBytes atomic.Int64 + +// SecretScanBytes returns the total content bytes scanned by shared ingestion. +func SecretScanBytes() int64 { + return secretScanBytes.Load() +} + +// ComputeSignalsAndSecrets derives aggregate signals and definite findings +// from the same retained message graph. +func ComputeSignalsAndSecrets( + session db.Session, messages []db.Message, +) (db.SessionSignalUpdate, []db.SecretFinding) { + update := ComputeSignalsFromMessages(session, messages) + findings, leak := ScanSecretsFromMessages( + session, messages, secrets.ScanDefinite, + ) + update.SecretLeakCount = leak + update.SecretsRulesVersion = secrets.DefiniteRulesVersion() + return update, findings +} + +// ComputeSignalsAndSecretsWithContentFailures uses precomputed staged result +// failures when placeholder content cannot carry the verdict. +func ComputeSignalsAndSecretsWithContentFailures( + session db.Session, messages []db.Message, failures map[string]bool, +) (db.SessionSignalUpdate, []db.SecretFinding) { + update := ComputeSignalsFromMessagesWithContentFailures( + session, messages, failures, + ) + findings, leak := ScanSecretsFromMessages( + session, messages, secrets.ScanDefinite, + ) + update.SecretLeakCount = leak + update.SecretsRulesVersion = secrets.DefiniteRulesVersion() + return update, findings +} + +// ComputeSignalsFromMessages derives session signals from normalized rows. +func ComputeSignalsFromMessages( + session db.Session, messages []db.Message, +) db.SessionSignalUpdate { + return ComputeSignalsFromToolRows( + session, messages, ExtractToolCallRows(messages), + ) +} + +// ComputeSignalsFromMessagesWithContentFailures derives signals using staged +// content-failure verdicts. +func ComputeSignalsFromMessagesWithContentFailures( + session db.Session, messages []db.Message, failures map[string]bool, +) db.SessionSignalUpdate { + rows := ExtractToolCallRows(messages) + PatchToolCallRowsWithContentFailures(rows, messages, failures) + return ComputeSignalsFromToolRows(session, messages, rows) +} + +// PatchToolCallRowsWithContentFailures applies staged verdicts to matching +// occurrence-qualified calls whose event status does not already decide them. +func PatchToolCallRowsWithContentFailures( + rows []signals.ToolCallRow, messages []db.Message, + failures map[string]bool, +) { + if len(failures) == 0 { + return + } + index := 0 + occurrences := make(map[string]int) + for _, message := range messages { + for _, call := range message.ToolCalls { + if index >= len(rows) { + return + } + failed := false + if call.ToolUseID != "" { + occurrence := occurrences[call.ToolUseID] + occurrences[call.ToolUseID] = occurrence + 1 + failed = failures[db.StagedToolCallKey( + call.ToolUseID, occurrence, + )] + } + if failed && rows[index].EventStatus == "" { + rows[index].ContentFailure = true + } + index++ + } + } +} + +// ComputeSignalsFromToolRows derives the complete aggregate signal update. +func ComputeSignalsFromToolRows( + session db.Session, messages []db.Message, rows []signals.ToolCallRow, +) db.SessionSignalUpdate { + heuristics := signals.AnalyzeHeuristics(signals.HeuristicInput{ + Messages: ExtractHeuristicMessages(messages), ToolRows: rows, + }) + contextTokens := ExtractContextTokens(messages) + boundaries := ExtractCompactBoundaryOrdinals(messages) + model := ExtractMostCommonModel(messages) + lastRole, _ := ExtractLastMessageRole(messages) + toolHealth := signals.ComputeToolHealth(rows) + contextPressure := signals.ComputeContextPressure( + contextTokens, session.PeakContextTokens, model, + ) + compactionCount := contextPressure.CompactionCount + if len(boundaries) > 0 { + compactionCount = len(boundaries) + } + ordinalRows := make([]signals.ToolCallOrdinal, 0, len(rows)) + for _, row := range rows { + ordinalRows = append(ordinalRows, signals.ToolCallOrdinal{ + MessageOrdinal: row.MessageOrdinal, ToolName: row.ToolName, + }) + } + midTaskCount := signals.CountMidTaskCompactions(boundaries, ordinalRows) + finalStreak := ComputeFinalStreak(rows) + hasContextData := session.HasPeakContextTokens + if !hasContextData { + for _, row := range contextTokens { + if row.HasContextTokens { + hasContextData = true + break + } + } + } + update := db.SessionSignalUpdate{ + ToolFailureSignalCount: toolHealth.FailureSignalCount, + ToolRetryCount: toolHealth.RetryCount, + EditChurnCount: toolHealth.EditChurnCount, + ConsecutiveFailureMax: toolHealth.ConsecutiveFailureMax, + EndedWithRole: lastRole, FinalFailureStreak: finalStreak, + CompactionCount: compactionCount, + MidTaskCompactionCount: midTaskCount, + ContextPressureMax: contextPressure.PressureMax, + HasToolCalls: len(rows) > 0, HasContextData: hasContextData, + QualitySignals: db.QualitySignals{ + Version: db.CurrentQualitySignalVersion, + ShortPromptCount: heuristics.ShortPromptCount, + UnstructuredStart: heuristics.UnstructuredStart, + MissingSuccessCriteriaCount: heuristics.MissingSuccessCriteriaCount, + MissingVerificationCount: heuristics.MissingVerificationCount, + DuplicatePromptCount: heuristics.DuplicatePromptCount, + NoCodeContextCount: heuristics.NoCodeContextCount, + RunawayToolLoopCount: heuristics.RunawayToolLoopCount, + }, + } + return RefreshSignalRecencyAt(session, messages, update, time.Now()) +} + +// RefreshSignalRecencyAt settles only clock-derived state using the same stable +// signal counters and retained transcript. It does not rescan secrets or tools. +func RefreshSignalRecencyAt(session db.Session, messages []db.Message, update db.SessionSignalUpdate, now time.Time) db.SessionSignalUpdate { + var lastActivity time.Time + if session.EndedAt != nil { + lastActivity, _ = time.Parse(time.RFC3339Nano, *session.EndedAt) + } + lastRole, lastContent := ExtractLastMessageRole(messages) + outcome := signals.ClassifyOutcomeAt(signals.OutcomeInput{ + IsAutomated: session.IsAutomated, MessageCount: session.MessageCount, + EndedWithRole: lastRole, FinalFailureStreak: update.FinalFailureStreak, + LastAssistantText: lastContent, LastActivity: lastActivity, + }, now) + q := update.QualitySignals + score := signals.ComputeHealthScore(signals.ScoreInput{ + Outcome: outcome.Outcome, OutcomeConfidence: outcome.Confidence, + HasToolCalls: update.HasToolCalls, FailureSignalCount: update.ToolFailureSignalCount, + RetryCount: update.ToolRetryCount, EditChurnCount: update.EditChurnCount, + ConsecutiveFailMax: update.ConsecutiveFailureMax, HasContextData: update.HasContextData, + CompactionCount: update.CompactionCount, MidTaskCompactionCount: update.MidTaskCompactionCount, + PressureMax: update.ContextPressureMax, Heuristics: signals.HeuristicSignals{ + ShortPromptCount: q.ShortPromptCount, UnstructuredStart: q.UnstructuredStart, + MissingSuccessCriteriaCount: q.MissingSuccessCriteriaCount, MissingVerificationCount: q.MissingVerificationCount, + DuplicatePromptCount: q.DuplicatePromptCount, NoCodeContextCount: q.NoCodeContextCount, + RunawayToolLoopCount: q.RunawayToolLoopCount, + }, + }) + update.Outcome, update.OutcomeConfidence = outcome.Outcome, outcome.Confidence + update.SignalsPendingSince = nil + if outcome.IsRecent { + pending := now.UTC().Format(time.RFC3339) + update.SignalsPendingSince = &pending + } + update.HealthScore, update.HealthGrade = score.Score, nil + if score.Grade != "" { + update.HealthGrade = &score.Grade + } + return update +} + +// ExtractToolCallRows builds the signal analyzer input from normalized calls. +func ExtractToolCallRows(messages []db.Message) []signals.ToolCallRow { + rows := make([]signals.ToolCallRow, 0) + for _, message := range messages { + for callIndex, call := range message.ToolCalls { + status := "" + if count := len(call.ResultEvents); count > 0 { + status = call.ResultEvents[count-1].Status + } + rows = append(rows, signals.ToolCallRow{ + ToolName: call.ToolName, Category: call.Category, + InputJSON: call.InputJSON, ResultContent: call.ResultContent, + MessageOrdinal: message.Ordinal, CallIndex: callIndex, + EventStatus: status, + }) + } + } + return rows +} + +// ComputeFinalStreak returns the trailing consecutive tool-failure count. +func ComputeFinalStreak(rows []signals.ToolCallRow) int { + streak := 0 + for _, row := range slices.Backward(rows) { + if !signals.IsFailure(row) { + break + } + streak++ + } + return streak +} + +// ScanSecretsFromMessages detects secrets in normalized retained content. +func ScanSecretsFromMessages( + _ db.Session, messages []db.Message, + scan func(string) []secrets.Match, +) (findings []db.SecretFinding, definiteCount int) { + findings = make([]db.SecretFinding, 0) + add := func( + sessionID, location string, ordinal int, callIndex, eventIndex *int, + content string, + ) { + secretScanBytes.Add(int64(len(content))) + for _, match := range scan(content) { + findings = append(findings, db.SecretFinding{ + SessionID: sessionID, RuleName: match.Rule, + Confidence: match.Confidence, LocationKind: location, + MessageOrdinal: ordinal, CallIndex: callIndex, + EventIndex: eventIndex, MatchStart: match.Start, + MatchEnd: match.End, MatchIndex: match.Index, + RedactedMatch: match.Redacted, + RulesVersion: secrets.DefiniteRulesVersion(), + }) + if match.Confidence == secrets.ConfidenceDefinite { + definiteCount++ + } + } + } + for _, message := range messages { + add(message.SessionID, "message", message.Ordinal, nil, nil, + message.Content) + for callIndex := range message.ToolCalls { + call := message.ToolCalls[callIndex] + ci := callIndex + add(message.SessionID, "tool_input", message.Ordinal, &ci, nil, + call.InputJSON) + if len(call.ResultEvents) > 0 { + for eventIndex := range call.ResultEvents { + ei := eventIndex + add(message.SessionID, "tool_result_event", message.Ordinal, + &ci, &ei, call.ResultEvents[eventIndex].Content) + } + } else { + add(message.SessionID, "tool_result", message.Ordinal, &ci, nil, + call.ResultContent) + } + } + } + return findings, definiteCount +} + +// ExtractHeuristicMessages builds prompt-quality analyzer inputs. +func ExtractHeuristicMessages(messages []db.Message) []signals.HeuristicMessage { + rows := make([]signals.HeuristicMessage, 0, len(messages)) + for _, message := range messages { + rows = append(rows, signals.HeuristicMessage{ + Role: message.Role, SourceSubtype: message.SourceSubtype, + Content: message.Content, IsSystem: message.IsSystem, + Ordinal: message.Ordinal, Timestamp: message.Timestamp, + }) + } + return rows +} + +// ExtractContextTokens returns assistant context measurements in order. +func ExtractContextTokens(messages []db.Message) []signals.ContextTokenRow { + var rows []signals.ContextTokenRow + for _, message := range messages { + if message.Role == "assistant" { + rows = append(rows, signals.ContextTokenRow{ + ContextTokens: message.ContextTokens, + HasContextTokens: message.HasContextTokens, + }) + } + } + return rows +} + +// ExtractCompactBoundaryOrdinals returns explicit compaction boundaries. +func ExtractCompactBoundaryOrdinals(messages []db.Message) []int { + var ordinals []int + for _, message := range messages { + if message.IsCompactBoundary { + ordinals = append(ordinals, message.Ordinal) + } + } + return ordinals +} + +// ExtractMostCommonModel returns the majority assistant model, with first-seen +// order breaking ties. +func ExtractMostCommonModel(messages []db.Message) string { + counts := make(map[string]int) + firstSeen := make(map[string]int) + for index, message := range messages { + if message.Role != "assistant" || message.Model == "" { + continue + } + counts[message.Model]++ + if _, exists := firstSeen[message.Model]; !exists { + firstSeen[message.Model] = index + } + } + best := "" + bestCount := -1 + for model, count := range counts { + if count > bestCount || + (count == bestCount && firstSeen[model] < firstSeen[best]) { + best, bestCount = model, count + } + } + return best +} + +// ExtractLastMessageRole returns the final visible transcript row. +func ExtractLastMessageRole(messages []db.Message) (role, content string) { + for _, message := range slices.Backward(messages) { + if !message.IsSystem && message.SourceSubtype != "tool_result" { + return message.Role, message.Content + } + } + return "", "" +} diff --git a/internal/ingest/tool_results.go b/internal/ingest/tool_results.go new file mode 100644 index 0000000000..570fa65d77 --- /dev/null +++ b/internal/ingest/tool_results.go @@ -0,0 +1,160 @@ +package ingest + +import ( + "context" + "strings" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +// PairAndFilterContext attaches result carriers to calls and removes empty +// carrier messages without renumbering the remaining transcript. +func PairAndFilterContext( + ctx context.Context, messages []db.Message, blocked map[string]bool, +) ([]db.Message, error) { + if err := PairToolResultsContext(ctx, messages, blocked); err != nil { + return nil, err + } + if err := PairToolResultEventSummariesContext( + ctx, messages, blocked, + ); err != nil { + return nil, err + } + filtered := messages[:0] + for _, message := range messages { + if err := ctx.Err(); err != nil { + return nil, err + } + if message.Role == "user" && len(message.ToolResults) > 0 && + strings.TrimSpace(message.Content) == "" { + continue + } + filtered = append(filtered, message) + } + return filtered, ctx.Err() +} + +// PairToolResultsContext attaches transient result carriers by tool-use ID. +func PairToolResultsContext( + ctx context.Context, messages []db.Message, blocked map[string]bool, +) error { + calls := make(map[string]*db.ToolCall) + for i := range messages { + if err := ctx.Err(); err != nil { + return err + } + for j := range messages[i].ToolCalls { + call := &messages[i].ToolCalls[j] + if call.ToolUseID != "" { + calls[call.ToolUseID] = call + } + } + } + for _, message := range messages { + if err := ctx.Err(); err != nil { + return err + } + for _, result := range message.ToolResults { + call, ok := calls[result.ToolUseID] + if !ok { + continue + } + call.ResultContentLength = result.ContentLength + if !blocked[call.Category] { + call.ResultContent = parser.DecodeContent(result.ContentRaw) + call.ResultContentLength = db.ResolveResultContentLength( + call.ResultContent, result.ContentLength, + ) + } + } + } + return ctx.Err() +} + +// PairToolResultEventSummariesContext derives the canonical call summary from +// structured result events. +func PairToolResultEventSummariesContext( + ctx context.Context, messages []db.Message, blocked map[string]bool, +) error { + for i := range messages { + if err := ctx.Err(); err != nil { + return err + } + for j := range messages[i].ToolCalls { + call := &messages[i].ToolCalls[j] + if len(call.ResultEvents) == 0 { + continue + } + summary, err := SummarizeToolResultEventsContext( + ctx, call.ResultEvents, + ) + if err != nil { + return err + } + call.ResultContentLength = len(summary) + if blocked[call.Category] { + call.ResultContent = "" + for k := range call.ResultEvents { + call.ResultEvents[k].Content = "" + } + continue + } + call.ResultContent = summary + } + } + return ctx.Err() +} + +// SummarizeToolResultEventsContext returns the latest content per agent while +// preserving first-seen agent order and the last anonymous update. +func SummarizeToolResultEventsContext( + ctx context.Context, events []db.ToolResultEvent, +) (string, error) { + if len(events) == 0 { + return "", ctx.Err() + } + type agentSummary struct { + content string + } + latestByAgent := make(map[string]agentSummary) + orderedAgents := make([]string, 0, len(events)) + lastAnonymous := "" + allHaveAgentID := true + for _, event := range events { + if err := ctx.Err(); err != nil { + return "", err + } + if strings.TrimSpace(event.Content) == "" { + continue + } + agentID := strings.TrimSpace(event.AgentID) + if agentID == "" { + allHaveAgentID = false + lastAnonymous = event.Content + continue + } + if _, exists := latestByAgent[agentID]; !exists { + orderedAgents = append(orderedAgents, agentID) + } + latestByAgent[agentID] = agentSummary{content: event.Content} + } + if len(latestByAgent) <= 1 { + if len(latestByAgent) == 1 { + summary := latestByAgent[orderedAgents[0]].content + if lastAnonymous != "" { + return summary + "\n\n" + lastAnonymous, ctx.Err() + } + return summary, ctx.Err() + } + return lastAnonymous, ctx.Err() + } + parts := make([]string, 0, len(orderedAgents)+1) + for _, agentID := range orderedAgents { + parts = append(parts, agentID+":\n"+latestByAgent[agentID].content) + } + if !allHaveAgentID && lastAnonymous != "" { + parts = append(parts, lastAnonymous) + } + return strings.Join(parts, "\n\n"), ctx.Err() +} diff --git a/internal/parser/claude_lineage.go b/internal/parser/claude_lineage.go index 6e7878840d..3d595149e5 100644 --- a/internal/parser/claude_lineage.go +++ b/internal/parser/claude_lineage.go @@ -53,8 +53,8 @@ const ( // re-reads of transcript heads. claudeSniffCacheMaxEntries = 8192 // claudeSniffDigestBytes bounds the transcript prefix hashed to - // verify sniff-cache entries without reliable change times. Size and - // mtime alone cannot detect a same-length rewrite that restores mtime, + // verify every metadata-matching sniff-cache entry. Size, mtime and + // ctime can all match after a rapid same-length rewrite, // so those hits require an unchanged leading digest too. // Heads whose parse runs past the bound are not memoized. claudeSniffDigestBytes = 64 << 10 @@ -71,7 +71,6 @@ type claudeHeadSniff struct { type claudeSniffCacheEntry struct { size int64 mtimeNS int64 - ctimeNS int64 headSHA [sha256.Size]byte sniff claudeHeadSniff } @@ -120,19 +119,14 @@ func claudeSniffHead( if err != nil || info.IsDir() { return claudeHeadSniff{}, nil } - ctimeNS, changeTimeVerified := codexIndexChangeTime(path, info) claudeSniffMu.Lock() e, cached := claudeSniffCache[path] claudeSniffMu.Unlock() if cached && e.size == info.Size() && e.mtimeNS == info.ModTime().UnixNano() { - if changeTimeVerified && ctimeNS != 0 && e.ctimeNS == ctimeNS { - return e.sniff, nil - } // Matching metadata does not prove matching content: a - // same-length rewrite that restores the mtime would replay a - // stale root uuid or bg stamp into lineage resolution. Verify - // a bounded digest of the leading bytes before trusting the - // memo. + // same-length rewrite can restore mtime while ctime also stays + // unchanged. Verify the bounded leading-byte digest on every + // metadata hit before reusing a root UUID or background stamp. headSHA, ok := claudeHeadDigest(path) if ok && headSHA == e.headSHA { return e.sniff, nil @@ -154,7 +148,6 @@ func claudeSniffHead( claudeSniffCache[path] = claudeSniffCacheEntry{ size: info.Size(), mtimeNS: info.ModTime().UnixNano(), - ctimeNS: ctimeNS, headSHA: headSHA, sniff: sniff, } @@ -354,8 +347,8 @@ func claudeLineageCaptureSiblings(ctx context.Context, path string) ([]string, e // claudeResolveSiblingLineage establishes the background-fork lineage // for path, or returns nil when no lineage can be positively oriented. -// Sibling discovery is head-sniff only (memoized per size, mtime, and -// change time, with a bounded digest when change time is unavailable). +// Sibling discovery is head-sniff only (memoized per size and mtime, +// with a bounded leading digest verifying every metadata hit). // Qualifying candidates' full uuid sets are read once per full parse of a // bg-marked fork. func claudeResolveSiblingLineage( diff --git a/internal/parser/claude_lineage_test.go b/internal/parser/claude_lineage_test.go index 4888957f32..1a90867751 100644 --- a/internal/parser/claude_lineage_test.go +++ b/internal/parser/claude_lineage_test.go @@ -1,6 +1,7 @@ package parser import ( + "crypto/sha256" "fmt" "os" "path/filepath" @@ -903,6 +904,35 @@ func TestClaudeLineageSniffCacheInvalidatedBySameSizeRewrite(t *testing.T) { } } +// Model a prior observation whose filesystem metadata collides with a newer +// same-size head. The full parser must validate bytes before using that memo. +func TestClaudeLineageSniffCacheRejectsStaleHeadWithMatchingMetadata(t *testing.T) { + t.Parallel() + original := lineageOriginalContent() + origPath, forkPath := writeLineageFixture(t, "orig-1111.jsonl", original, "fork-2222.jsonl", lineageForkContent()) + warm, _, err := claudeParseFile(forkPath, "my_app", "local", claudeParseOptions{siblingLineage: true}) + require.NoError(t, err) + require.Len(t, warm, 1) + require.Equal(t, "orig-1111", warm[0].Session.ParentSessionID) + // This is the memo a previous z9 head can leave when stat metadata has + // coarse resolution. Retain all actual observed cache metadata. + prior := strings.ReplaceAll(original, `"u1"`, `"z9"`) + require.Len(t, prior, len(original)) + claudeSniffMu.Lock() + cached, ok := claudeSniffCache[origPath] + cached.headSHA = sha256.Sum256([]byte(prior)) + cached.sniff.rootUUID = "z9" + claudeSniffCache[origPath] = cached + claudeSniffMu.Unlock() + require.True(t, ok, "full parse must warm the original's head") + results, _, err := claudeParseFile(forkPath, "my_app", "local", claudeParseOptions{siblingLineage: true}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "orig-1111", results[0].Session.ParentSessionID) + require.Len(t, results[0].Messages, 2) + assert.Equal(t, "continued question", firstMessageContent(results[0].Messages)) +} + func BenchmarkClaudeLineageSniffCacheHit(b *testing.B) { path := filepath.Join(b.TempDir(), "session.jsonl") content := lineageOriginalContent() + strings.Repeat(" ", 64<<10) diff --git a/internal/parser/types.go b/internal/parser/types.go index 34769b7519..27e75d790a 100644 --- a/internal/parser/types.go +++ b/internal/parser/types.go @@ -1701,3 +1701,20 @@ func InferRelationshipTypes(results []ParseResult) { } } } + +// TokenPresenceKnown reports whether provider flags override legacy inference. +// Transport adapters must preserve it alongside the exported token fields. +func (m ParsedMessage) TokenPresenceKnown() bool { return m.tokenPresenceKnown } + +// RestoreTokenPresenceKnown restores transport metadata without inferring or +// changing the provider's exported token values and coverage flags. +func (m *ParsedMessage) RestoreTokenPresenceKnown(known bool) { m.tokenPresenceKnown = known } + +// AggregateTokenPresenceKnown reports whether session aggregate coverage is +// authoritative rather than inferred from nonzero legacy values. +func (s ParsedSession) AggregateTokenPresenceKnown() bool { return s.aggregateTokenPresenceKnown } + +// RestoreAggregateTokenPresenceKnown restores the session transport metadata. +func (s *ParsedSession) RestoreAggregateTokenPresenceKnown(known bool) { + s.aggregateTokenPresenceKnown = known +} diff --git a/internal/postgres/connect.go b/internal/postgres/connect.go index 94828d7778..42890ce1fa 100644 --- a/internal/postgres/connect.go +++ b/internal/postgres/connect.go @@ -13,8 +13,9 @@ import ( "strings" "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" - _ "github.com/jackc/pgx/v5/stdlib" + "github.com/jackc/pgx/v5/stdlib" ) // RedactDSN returns the host portion of the DSN for diagnostics, @@ -263,3 +264,52 @@ func appendConnParams( } return result, nil } + +// OpenHosted opens and verifies a restricted pool permanently assigned to one +// tenant schema. Startup parameters override caller values on every connection; +// checkout reset restores context after connection reuse or caller SET commands. +func OpenHosted(dsn, schema, tenant string, allowInsecure bool) (*sql.DB, error) { + if err := validateHostedBinding(schema, tenant); err != nil { + return nil, err + } + if dsn == "" { + return nil, fmt.Errorf("postgres URL is required") + } + if allowInsecure { + WarnInsecureSSL(dsn) + } else if err := CheckSSL(dsn); err != nil { + return nil, err + } + quoted, _ := quoteIdentifier(schema) + connStr, err := appendConnParams(dsn, map[string]string{"search_path": quoted, "TimeZone": "UTC"}) + if err != nil { + return nil, err + } + cfg, err := pgx.ParseConfig(connStr) + if err != nil { + return nil, err + } + // libpq options and role parameters must not override our startup boundary. + delete(cfg.RuntimeParams, "options") + delete(cfg.RuntimeParams, "role") + // Naming pg_temp explicitly prevents its otherwise implicit first priority. + searchPath := quoted + ", pg_temp" + cfg.RuntimeParams["search_path"] = searchPath + cfg.RuntimeParams["agentsview.tenant_id"] = tenant + cfg.RuntimeParams["row_security"] = "on" + database := stdlib.OpenDB(*cfg, stdlib.OptionResetSession(func(ctx context.Context, c *pgx.Conn) error { + _, err := c.Exec(ctx, `SELECT set_config('search_path',$1,false),set_config('agentsview.tenant_id',$2,false),set_config('row_security','on',false)`, searchPath, tenant) + return err + })) + database.SetMaxOpenConns(5) + database.SetMaxIdleConns(5) + database.SetConnMaxLifetime(30 * time.Minute) + database.SetConnMaxIdleTime(5 * time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err = CheckHostedTenant(ctx, database, schema, tenant); err != nil { + database.Close() + return nil, err + } + return database, nil +} diff --git a/internal/postgres/curation.go b/internal/postgres/curation.go index 5165038177..1433cf8c9f 100644 --- a/internal/postgres/curation.go +++ b/internal/postgres/curation.go @@ -151,8 +151,19 @@ func (s *Store) PinMessage( return 0, err } + id, err := pinMessageTx(ctx, tx, sessionID, messageID, note) + if err != nil { + return 0, err + } + if err = tx.Commit(); err != nil { + return 0, err + } + return id, nil +} + +func pinMessageTx(ctx context.Context, tx *sql.Tx, sessionID string, messageID int64, note *string) (int64, error) { var id int64 - err = tx.QueryRowContext(ctx, ` + err := tx.QueryRowContext(ctx, ` WITH upsert AS ( INSERT INTO pinned_messages ( session_id, message_id, ordinal, source_uuid, note @@ -172,18 +183,9 @@ func (s *Store) PinMessage( sessionID, messageID, note, ).Scan(&id) if err == sql.ErrNoRows { - if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("committing empty pin transaction: %w", err) - } return 0, nil } - if err != nil { - return 0, fmt.Errorf("pinning message: %w", err) - } - if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("committing pin transaction: %w", err) - } - return id, nil + return id, err } // UnpinMessage removes a shared PG pin. diff --git a/internal/postgres/hosted_curation.go b/internal/postgres/hosted_curation.go new file mode 100644 index 0000000000..cb32d2ac3d --- /dev/null +++ b/internal/postgres/hosted_curation.go @@ -0,0 +1,236 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "go.kenn.io/agentsview/internal/db" + "slices" +) + +func hostedCurationError(err error) error { + if e, ok := errors.AsType[*RawIdentityError](err); ok { + return &db.SessionIdentityError{State: string(e.Identity.State), Variants: e.Identity.Variants} + } + return err +} +func (h *HostedStore) curate(alias, field string, value any) error { + ctx := context.Background() + target, err := h.resolve(ctx, alias) + if err != nil { + return err + } + if target.Legacy { + _, err = h.legacyWrite(ctx, alias, target.SessionID, func(tx *sql.Tx, id string) (int64, error) { return legacyCurationTx(ctx, tx, id, field, value) }) + return err + } + return hostedCurationError(h.core.SetCuration(ctx, alias, field, value)) +} +func (h *HostedStore) RenameSession(id string, name *string) error { + var value any + if name != nil { + value = *name + } + return h.curate(id, "display_name", value) +} +func (h *HostedStore) SoftDeleteSession(id string) error { + return h.curate(id, "trashed", true) +} +func (h *HostedStore) RestoreSession(id string) (int64, error) { + target, err := h.resolve(context.Background(), id) + if err != nil { + return 0, err + } + if target.Legacy { + return h.legacyWrite(context.Background(), id, target.SessionID, func(tx *sql.Tx, p string) (int64, error) { + return legacyCurationTx(context.Background(), tx, p, "trashed", false) + }) + } + err = hostedCurationError(h.core.SetCuration(context.Background(), id, "trashed", false)) + if err != nil { + return 0, err + } + return 1, nil +} +func (h *HostedStore) StarSession(id string) (bool, error) { + err := h.curate(id, "starred", true) + return err == nil, err +} +func (h *HostedStore) UnstarSession(id string) error { + return h.curate(id, "starred", false) +} +func (h *HostedStore) BulkStarSessions(ids []string) error { + if _, err := h.resolveBatch(context.Background(), ids, false, true); err != nil { + return err + } + for _, id := range ids { + if _, err := h.StarSession(id); err != nil { + return err + } + } + return nil +} +func (h *HostedStore) SoftDeleteSessions(ids []string) (int, error) { + if _, err := h.resolveBatch(context.Background(), ids, false, true); err != nil { + return 0, err + } + n := 0 + for _, id := range ids { + if err := h.SoftDeleteSession(id); err != nil { + return n, err + } + n++ + } + return n, nil +} +func (h *HostedStore) ListStarredSessionIDs(ctx context.Context) ([]string, error) { + return hostedMapped(ctx, h, func() ([]string, error) { return h.physical.ListStarredSessionIDs(ctx) }, func(p *[]string, r *hostedRefs) { + *p = slices.Clone(*p) + for i := range *p { + r.add(&(*p)[i]) + } + }) +} +func (h *HostedStore) ListTrashedSessions(ctx context.Context) ([]db.Session, error) { + return hostedMapped(ctx, h, func() ([]db.Session, error) { return h.physical.ListTrashedSessions(ctx) }, func(p *[]db.Session, r *hostedRefs) { + *p = slices.Clone(*p) + for i := range *p { + r.session(&(*p)[i]) + } + }) +} +func (h *HostedStore) DeleteSessionIfTrashed(id string) (int64, error) { + ctx := context.Background() + target, err := h.resolve(ctx, id) + if err != nil { + return 0, err + } + if target.Legacy { + return h.legacyWrite(ctx, id, target.SessionID, func(tx *sql.Tx, p string) (int64, error) { return deleteLegacyTrashedTx(ctx, tx, p) }) + } + deleted, err := h.core.ExcludeTrashedSession(ctx, id) + if err != nil { + return 0, hostedCurationError(err) + } + if deleted { + return 1, nil + } + return 0, nil +} +func (h *HostedStore) EmptyTrash() (int, error) { + n, err := h.core.EmptyTrash(context.Background()) + if err != nil { + return n, err + } + legacy, err := h.physical.emptyTrash(true) + return n + legacy, err +} +func (h *HostedStore) PinMessage(id string, ordinal int64, note *string) (int64, error) { + ctx := context.Background() + target, err := h.resolve(ctx, id) + if err != nil { + return 0, err + } + if target.Legacy { + return h.legacyWrite(ctx, id, target.SessionID, func(tx *sql.Tx, p string) (int64, error) { return pinMessageTx(ctx, tx, p, ordinal, note) }) + } + text := "" + if note != nil { + text = *note + } + pinID, err := h.core.SetPinReturningID(ctx, id, int(ordinal), true, text) + return pinID, hostedCurationError(err) +} +func (h *HostedStore) UnpinMessage(id string, ordinal int64) error { + ctx := context.Background() + target, err := h.resolve(ctx, id) + if err != nil { + return err + } + if target.Legacy { + _, err = h.legacyWrite(ctx, id, target.SessionID, func(tx *sql.Tx, p string) (int64, error) { + res, e := tx.ExecContext(ctx, `DELETE FROM pinned_messages WHERE session_id=$1 AND message_id=$2`, p, ordinal) + if e != nil { + return 0, e + } + return res.RowsAffected() + }) + return err + } + return hostedCurationError(h.core.SetPin(ctx, id, int(ordinal), false, "")) +} +func (h *HostedStore) RemovePinReference(ctx context.Context, id, key string) error { + target, err := h.resolve(ctx, id) + if err != nil { + return err + } + if target.Legacy { + return db.ErrReadOnly + } + return hostedCurationError(h.core.RemovePin(ctx, id, key)) +} +func (h *HostedStore) ListPinnedMessages(ctx context.Context, id, project string) ([]db.PinnedMessage, error) { + return hostedRead(ctx, h, func(_ hostedRevision) ([]db.PinnedMessage, error) { + physical := "" + var target hostedIdentity + var err error + if id != "" { + target, err = h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + physical = target.SessionID + } + pins, err := h.core.pinReferencePage(ctx, physical, project) + if err != nil { + return nil, err + } + r := hostedRefs{} + for i := range pins { + r.add(&pins[i].SessionID) + } + if err = r.mapIDs(ctx, h); err != nil { + return nil, err + } + return pins, nil + }) +} +func (h *HostedStore) GetSessionVersion(id string) (int, int64, bool) { + target, err := h.resolve(context.Background(), id) + if err != nil || target.SessionID == "" { + return 0, 0, false + } + return h.physical.GetSessionVersion(target.SessionID) +} +func (h *HostedStore) GetSessionWatchState(id string) (db.SessionWatchState, error) { + return hostedRead(context.Background(), h, func(r hostedRevision) (db.SessionWatchState, error) { + target, err := h.resolve(context.Background(), id) + if conflict, ok := errors.AsType[*db.SessionIdentityError](err); ok { + return db.SessionWatchState{State: conflict.State, Variants: conflict.Variants, IdentityRevision: r.Identity}, nil + } + if err != nil { + return db.SessionWatchState{}, err + } + state := string(target.State) + if target.State == RawIdentityUnique { + state = db.SessionWatchResolved + } + return db.SessionWatchState{State: state, PublicID: target.PublicID, IdentityRevision: r.Identity, ContentRevision: target.ContentRevision}, nil + }) +} + +// Recheck public legacy collisions inside the core's existing group lock and +// write transaction. Legacy membership writes take the same lock in their trigger. +func (h *HostedStore) guardCurationIdentity(ctx context.Context, tx *sql.Tx, alias string, target RawIdentity) error { + var legacy string + err := tx.QueryRowContext(ctx, `SELECT hosted_legacy_alias(id) FROM sessions WHERE provenance_kind='legacy' AND (id=$1 OR hosted_legacy_alias(id)=$1)`, alias).Scan(&legacy) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + variants := append(slices.Clone(target.Variants), legacy) + slices.Sort(variants) + return &db.SessionIdentityError{State: "ambiguous", Variants: variants} +} diff --git a/internal/postgres/hosted_forward.go b/internal/postgres/hosted_forward.go new file mode 100644 index 0000000000..db23224b75 --- /dev/null +++ b/internal/postgres/hosted_forward.go @@ -0,0 +1,135 @@ +package postgres + +import ( + "context" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/export" + "io" +) + +func (h *HostedStore) HasFTS() bool { return h.physical.HasFTS() } +func (h *HostedStore) HasSemantic() bool { return h.physical.HasSemantic() } +func (h *HostedStore) GetStats(ctx context.Context, excludeOneShot, excludeAutomated bool) (db.Stats, error) { + return h.physical.GetStats(ctx, excludeOneShot, excludeAutomated) +} +func (h *HostedStore) GetProjects(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]db.ProjectInfo, error) { + return h.physical.GetProjects(ctx, excludeOneShot, excludeAutomated) +} +func (h *HostedStore) GetActiveProjectLabels(ctx context.Context) ([]string, error) { + return h.physical.GetActiveProjectLabels(ctx) +} +func (h *HostedStore) GetAgents(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]db.AgentInfo, error) { + return h.physical.GetAgents(ctx, excludeOneShot, excludeAutomated) +} +func (h *HostedStore) GetMachines(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]string, error) { + return h.physical.GetMachines(ctx, excludeOneShot, excludeAutomated) +} +func (h *HostedStore) GetBranches(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]db.BranchInfo, error) { + return h.physical.GetBranches(ctx, excludeOneShot, excludeAutomated) +} +func (h *HostedStore) ListProjectIdentityObservations(ctx context.Context, labels []string) ([]export.ProjectIdentityObservation, error) { + return h.physical.ListProjectIdentityObservations(ctx, labels) +} +func (h *HostedStore) BuildProjectIdentityMap(ctx context.Context, labels []string) (map[string]export.ProjectMapEntry, error) { + return h.physical.BuildProjectIdentityMap(ctx, labels) +} +func (h *HostedStore) GetProjectInventory(ctx context.Context) (db.ProjectInventory, error) { + return h.physical.GetProjectInventory(ctx) +} +func (h *HostedStore) ListProjectRules(ctx context.Context, machine string) (db.ProjectRules, error) { + return h.physical.ListProjectRules(ctx, machine) +} +func (h *HostedStore) GetAnalyticsSummary(ctx context.Context, f db.AnalyticsFilter) (db.AnalyticsSummary, error) { + return h.physical.GetAnalyticsSummary(ctx, f) +} +func (h *HostedStore) GetAnalyticsActivity(ctx context.Context, f db.AnalyticsFilter, granularity string) (db.ActivityResponse, error) { + return h.physical.GetAnalyticsActivity(ctx, f, granularity) +} +func (h *HostedStore) GetAnalyticsHeatmap(ctx context.Context, f db.AnalyticsFilter, metric string) (db.HeatmapResponse, error) { + return h.physical.GetAnalyticsHeatmap(ctx, f, metric) +} +func (h *HostedStore) GetAnalyticsProjects(ctx context.Context, f db.AnalyticsFilter) (db.ProjectsAnalyticsResponse, error) { + return h.physical.GetAnalyticsProjects(ctx, f) +} +func (h *HostedStore) GetAnalyticsHourOfWeek(ctx context.Context, f db.AnalyticsFilter) (db.HourOfWeekResponse, error) { + return h.physical.GetAnalyticsHourOfWeek(ctx, f) +} +func (h *HostedStore) GetAnalyticsSessionShape(ctx context.Context, f db.AnalyticsFilter) (db.SessionShapeResponse, error) { + return h.physical.GetAnalyticsSessionShape(ctx, f) +} +func (h *HostedStore) GetAnalyticsTools(ctx context.Context, f db.AnalyticsFilter) (db.ToolsAnalyticsResponse, error) { + return h.physical.GetAnalyticsTools(ctx, f) +} +func (h *HostedStore) GetAnalyticsSkills(ctx context.Context, f db.AnalyticsFilter, granularity string) (db.SkillsAnalyticsResponse, error) { + return h.physical.GetAnalyticsSkills(ctx, f, granularity) +} +func (h *HostedStore) GetAnalyticsVelocity(ctx context.Context, f db.AnalyticsFilter) (db.VelocityResponse, error) { + return h.physical.GetAnalyticsVelocity(ctx, f) +} +func (h *HostedStore) GetAnalyticsSignals(ctx context.Context, f db.AnalyticsFilter) (db.SignalsAnalyticsResponse, error) { + return h.physical.GetAnalyticsSignals(ctx, f) +} +func (h *HostedStore) GetTrendsTerms(ctx context.Context, f db.AnalyticsFilter, terms []db.TrendTermInput, granularity string) (db.TrendsTermsResponse, error) { + return h.physical.GetTrendsTerms(ctx, f, terms, granularity) +} +func (h *HostedStore) GetDailyUsage(ctx context.Context, f db.UsageFilter) (db.DailyUsageResult, error) { + return h.physical.GetDailyUsage(ctx, f) +} +func (h *HostedStore) GetUsageSessionCounts(ctx context.Context, f db.UsageFilter) (db.UsageSessionCounts, error) { + return h.physical.GetUsageSessionCounts(ctx, f) +} +func (h *HostedStore) GetUsageMatchingSessionCount(ctx context.Context, f db.UsageFilter) (int, error) { + return h.physical.GetUsageMatchingSessionCount(ctx, f) +} +func (h *HostedStore) ListInsights(ctx context.Context, f db.InsightFilter) ([]db.Insight, error) { + return h.physical.ListInsights(ctx, f) +} +func (h *HostedStore) GetInsight(ctx context.Context, id int64) (*db.Insight, error) { + return h.physical.GetInsight(ctx, id) +} +func (h *HostedStore) GetCachedInsight(ctx context.Context, cacheKey string) (*db.Insight, error) { + return h.physical.GetCachedInsight(ctx, cacheKey) +} +func (h *HostedStore) InsertInsight(s db.Insight) (int64, error) { return h.physical.InsertInsight(s) } +func (h *HostedStore) DeleteInsight(id int64) error { return h.physical.DeleteInsight(id) } +func (h *HostedStore) ListRecallEntries(ctx context.Context, q db.RecallQuery) ([]db.RecallEntry, error) { + return h.physical.ListRecallEntries(ctx, q) +} +func (h *HostedStore) GetRecallEntry(ctx context.Context, id string) (*db.RecallEntry, error) { + return h.physical.GetRecallEntry(ctx, id) +} +func (h *HostedStore) QueryRecallEntries(ctx context.Context, q db.RecallQuery) (db.RecallPage, error) { + return h.physical.QueryRecallEntries(ctx, q) +} +func (h *HostedStore) RecordRecallQueryEvent( + ctx context.Context, event db.RecallQueryEvent, +) (string, error) { + return h.physical.RecordRecallQueryEvent(ctx, event) +} +func (h *HostedStore) InsertRecallEntry(m db.RecallEntry) (string, error) { + return h.physical.InsertRecallEntry(m) +} +func (h *HostedStore) ImportAcceptedRecallEntriesJSONL(ctx context.Context, r io.Reader) (db.RecallImportResult, error) { + return h.physical.ImportAcceptedRecallEntriesJSONL(ctx, r) +} +func (h *HostedStore) ImportAcceptedRecallEntriesJSONLWithOptions( + ctx context.Context, r io.Reader, opts db.RecallImportOptions, +) (db.RecallImportResult, error) { + return h.physical.ImportAcceptedRecallEntriesJSONLWithOptions(ctx, r, opts) +} +func (h *HostedStore) IngestEvalTrajectory( + ctx context.Context, in db.EvalTrajectoryIngest, +) (db.EvalTrajectoryIngestResult, error) { + return h.physical.IngestEvalTrajectory(ctx, in) +} +func (h *HostedStore) UpsertSession(s db.Session) error { return h.physical.UpsertSession(s) } +func (h *HostedStore) ReplaceSessionMessages(sessionID string, msgs []db.Message) error { + return h.physical.ReplaceSessionMessages(sessionID, msgs) +} +func (h *HostedStore) WriteSessionBatchAtomic( + writes []db.SessionBatchWrite, + beforeCommit ...func() error, +) (db.SessionBatchResult, error) { + return h.physical.WriteSessionBatchAtomic(writes, beforeCommit...) +} +func (h *HostedStore) ReadOnly() bool { return h.physical.ReadOnly() } diff --git a/internal/postgres/hosted_legacy_curation.go b/internal/postgres/hosted_legacy_curation.go new file mode 100644 index 0000000000..1dff235f2c --- /dev/null +++ b/internal/postgres/hosted_legacy_curation.go @@ -0,0 +1,154 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + + "go.kenn.io/agentsview/internal/db" +) + +func legacyPublicVariant(id string) string { + digest := sha256.Sum256([]byte(id)) + return "legacy~" + hex.EncodeToString(digest[:]) +} + +// Alias locks fence first publication before a raw group exists. A fixed 256 +// schema-scoped buckets bounds shared-lock memory for large legacy batches. +// Collisions only add contention; acquire numeric bucket order before any group +// or session lock, and never call this helper from a raw group guard. +func lockHostedAliases(ctx context.Context, tx *sql.Tx, aliases []string) error { + rows, err := tx.QueryContext(ctx, `SELECT DISTINCT (hashtextextended(alias,0)&255)::int AS bucket FROM unnest($1::text[]) alias ORDER BY bucket`, aliases) + if err != nil { + return err + } + var buckets []int + for rows.Next() { + var bucket int + if err = rows.Scan(&bucket); err != nil { + rows.Close() + return err + } + buckets = append(buckets, bucket) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, bucket := range buckets { + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtext(current_schema()),$1)`, bucket); err != nil { + return err + } + } + return nil +} + +func (h *HostedStore) legacyWrite(ctx context.Context, alias, id string, write func(*sql.Tx, string) (int64, error)) (int64, error) { + tx, err := h.physical.pg.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + aliases := []string{id, legacyPublicVariant(id), alias} + if err = lockHostedAliases(ctx, tx, aliases); err != nil { + return 0, err + } + rows, err := tx.QueryContext(ctx, `SELECT g.group_id FROM raw_session_groups g WHERE EXISTS(SELECT 1 FROM raw_session_public_aliases a WHERE a.group_id=g.group_id AND a.alias_id=ANY($1)) ORDER BY g.group_id FOR UPDATE OF g`, aliases) + if err != nil { + return 0, err + } + for rows.Next() { + var group string + if err = rows.Scan(&group); err != nil { + rows.Close() + return 0, err + } + } + err = rows.Err() + rows.Close() + if err != nil { + return 0, err + } + var current string + err = tx.QueryRowContext(ctx, `SELECT id FROM sessions WHERE provenance_kind='legacy' AND id=$1 AND (id=$2 OR hosted_legacy_alias(id)=$2) FOR UPDATE`, id, alias).Scan(¤t) + if err == sql.ErrNoRows { + return 0, &db.SessionIdentityError{State: "gone"} + } + if err != nil { + return 0, err + } + raw, err := resolveRawIdentity(ctx, tx, alias) + if err != nil { + return 0, err + } + if raw.State != RawIdentityGone { + return 0, &db.SessionIdentityError{State: "ambiguous", Variants: append(raw.Variants, legacyPublicVariant(id))} + } + n, err := write(tx, current) + if err != nil { + return 0, err + } + // Star/pin tables do not fire the sessions revision trigger. + if _, err = tx.ExecContext(ctx, `UPDATE raw_corpus_state SET corpus_revision=corpus_revision+1 WHERE singleton=1`); err != nil { + return 0, err + } + if err = tx.Commit(); err != nil { + return 0, err + } + return n, nil +} + +func legacyCurationTx(ctx context.Context, tx *sql.Tx, id, field string, value any) (int64, error) { + var result sql.Result + var err error + switch field { + case "display_name": + result, err = tx.ExecContext(ctx, `UPDATE sessions SET display_name=$2,updated_at=NOW() WHERE id=$1 AND deleted_at IS NULL`, id, value) + case "starred": + if value.(bool) { + result, err = tx.ExecContext(ctx, `INSERT INTO starred_sessions(session_id) VALUES($1) ON CONFLICT(session_id) DO NOTHING`, id) + } else { + result, err = tx.ExecContext(ctx, `DELETE FROM starred_sessions WHERE session_id=$1`, id) + } + case "trashed": + if value.(bool) { + result, err = tx.ExecContext(ctx, `UPDATE sessions SET deleted_at=NOW(),deletion_cause=NULL,updated_at=NOW() WHERE id=$1 AND deleted_at IS NULL`, id) + } else { + result, err = tx.ExecContext(ctx, `UPDATE sessions SET deleted_at=NULL,deletion_cause=NULL,data_version=$2,updated_at=NOW() WHERE id=$1 AND deleted_at IS NOT NULL`, id, max(db.CurrentDataVersion()-1, 0)) + } + default: + return 0, fmt.Errorf("unsupported legacy curation field %q", field) + } + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +func deleteLegacyTrashedTx(ctx context.Context, tx *sql.Tx, id string) (int64, error) { + ids, excluded, err := readPGTrashedSessionExclusions(ctx, tx, `s.id=$1 AND s.provenance_kind='legacy' AND s.deleted_at IS NOT NULL`, id) + if err != nil || len(ids) == 0 { + return 0, err + } + if err = insertPGExcludedSessionIDs(ctx, tx, excluded); err != nil { + return 0, err + } + n, err := deletePGTrashedSessionRows(ctx, tx, ids) + if err != nil { + return 0, err + } + if err = deleteLegacyExcludedSessionRows(ctx, tx, excluded); err != nil { + return 0, err + } + return n, nil +} + +// Legacy alias cleanup must never reach a raw row, even if an adopted archive +// has an alias spelling that later collides with a physical projection identity. +func deleteLegacyExcludedSessionRows(ctx context.Context, tx *sql.Tx, ids []string) error { + _, err := tx.ExecContext(ctx, `DELETE FROM sessions WHERE provenance_kind='legacy' AND id=ANY($1)`, ids) + return err +} diff --git a/internal/postgres/hosted_legacy_revision.go b/internal/postgres/hosted_legacy_revision.go new file mode 100644 index 0000000000..698d6f614c --- /dev/null +++ b/internal/postgres/hosted_legacy_revision.go @@ -0,0 +1,40 @@ +package postgres + +// Triggers run after sessions row locks. Membership changes therefore never +// wait for an alias/group lock: contention aborts atomically with SQLSTATE 40001 +// and the owner must retry the complete write. Ordinary metadata only bumps +// corpus revision and does not acquire identity locks. +const hostedLegacyRevisionBody = ` +DECLARE old_legacy boolean := false; new_legacy boolean := false; identity_change boolean; bound_tenant text; legacy_ids text[] := '{}'; lock_bucket integer; +BEGIN + IF TG_OP <> 'INSERT' THEN old_legacy := OLD.provenance_kind='legacy' AND COALESCE(OLD.owner_marker,'') NOT LIKE 'raw-projection:%'; END IF; + IF TG_OP <> 'DELETE' THEN new_legacy := NEW.provenance_kind='legacy' AND COALESCE(NEW.owner_marker,'') NOT LIKE 'raw-projection:%'; END IF; + IF NOT old_legacy AND NOT new_legacy THEN RETURN NULL; END IF; + IF TG_OP='DELETE' THEN bound_tenant:=OLD.tenant_id; ELSE bound_tenant:=NEW.tenant_id; END IF; + identity_change := TG_OP <> 'UPDATE'; + IF TG_OP='UPDATE' THEN identity_change := old_legacy IS DISTINCT FROM new_legacy OR OLD.id IS DISTINCT FROM NEW.id; END IF; + IF identity_change THEN + IF old_legacy THEN legacy_ids:=array_append(legacy_ids,OLD.id); legacy_ids:=array_append(legacy_ids,hosted_legacy_alias(OLD.id)); END IF; + IF new_legacy THEN legacy_ids:=array_append(legacy_ids,NEW.id); legacy_ids:=array_append(legacy_ids,hosted_legacy_alias(NEW.id)); END IF; + FOR lock_bucket IN SELECT DISTINCT (hashtextextended(a,0)&255)::int FROM unnest(legacy_ids) a ORDER BY 1 LOOP + IF NOT pg_try_advisory_xact_lock(hashtext(TG_TABLE_SCHEMA),lock_bucket) THEN + RAISE EXCEPTION 'hosted legacy identity is busy; retry transaction' USING ERRCODE='40001'; + END IF; + END LOOP; + BEGIN + PERFORM g.group_id FROM raw_session_groups g WHERE EXISTS(SELECT 1 FROM raw_session_public_aliases a WHERE a.group_id=g.group_id AND a.alias_id=ANY(legacy_ids)) ORDER BY g.group_id FOR UPDATE OF g NOWAIT; + EXCEPTION WHEN lock_not_available THEN + RAISE EXCEPTION 'hosted legacy identity is busy; retry transaction' USING ERRCODE='40001'; + END; + END IF; + INSERT INTO raw_corpus_state(tenant_id,singleton,identity_revision,corpus_revision) VALUES(bound_tenant,1,CASE WHEN identity_change THEN 1 ELSE 0 END,1) + ON CONFLICT(tenant_id,singleton) DO UPDATE SET identity_revision=raw_corpus_state.identity_revision+EXCLUDED.identity_revision,corpus_revision=raw_corpus_state.corpus_revision+1; + RETURN NULL; +END; ` +const hostedLegacyAliasBody = ` SELECT 'legacy~'||encode(sha256(convert_to($1,'UTF8')),'hex') ` +const hostedLegacyRevisionDDL = ` +CREATE OR REPLACE FUNCTION hosted_legacy_alias(text) RETURNS text LANGUAGE sql IMMUTABLE STRICT AS $alias$` + hostedLegacyAliasBody + `$alias$; +CREATE OR REPLACE FUNCTION hosted_legacy_revision() RETURNS trigger LANGUAGE plpgsql AS $revision$` + hostedLegacyRevisionBody + `$revision$; +CREATE OR REPLACE TRIGGER hosted_legacy_revision AFTER INSERT OR UPDATE OR DELETE ON sessions FOR EACH ROW EXECUTE FUNCTION hosted_legacy_revision(); +CREATE INDEX IF NOT EXISTS hosted_legacy_alias ON sessions(tenant_id,(hosted_legacy_alias(id))) WHERE provenance_kind='legacy'; +` diff --git a/internal/postgres/hosted_public_export_test.go b/internal/postgres/hosted_public_export_test.go new file mode 100644 index 0000000000..1e8cb3e8c3 --- /dev/null +++ b/internal/postgres/hosted_public_export_test.go @@ -0,0 +1,26 @@ +//go:build pgtest + +package postgres + +import ( + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "testing" +) + +// HostedPublicFixture is shared only with the external service/HTTP tests. +func HostedPublicFixture(t *testing.T) (db.Store, func(), func()) { + t.Helper() + f := newProjectionFixture(t) + a, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("readable transcript"))) + h, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, h.Close()) }) + split := func() { + b, _ := f.accept(t, "device-b", "capture-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("other transcript"))) + } + pin := func() { _, err := h.PinMessage("codex:portable", 0, new("remember this")); require.NoError(t, err) } + return h, split, pin +} diff --git a/internal/postgres/hosted_public_http_pgtest_test.go b/internal/postgres/hosted_public_http_pgtest_test.go new file mode 100644 index 0000000000..9199ba7027 --- /dev/null +++ b/internal/postgres/hosted_public_http_pgtest_test.go @@ -0,0 +1,100 @@ +//go:build pgtest + +package postgres_test + +import ( + "context" + "encoding/json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/postgres" + "go.kenn.io/agentsview/internal/server" + "go.kenn.io/agentsview/internal/service" + "go.kenn.io/agentsview/internal/sessionwatch" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestHostedServiceAndHTTPPublicBoundary(t *testing.T) { + store, split, pin := postgres.HostedPublicFixture(t) + backend := service.NewReadOnlyBackend(store) + session, err := backend.Get(t.Context(), "codex:portable") + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "codex:portable", session.ID) + cfg, err := config.Default() + require.NoError(t, err) + cfg.DataDir = t.TempDir() + app := server.New(cfg, store, nil) + request := func(path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Host = "localhost:8080" + w := httptest.NewRecorder() + app.Handler().ServeHTTP(w, req) + return w + } + for _, path := range []string{"/api/v1/sessions/codex:portable", "/api/v1/sessions/codex:portable/messages", "/api/v1/sessions/codex:portable/export"} { + w := request(path) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + if strings.HasSuffix(path, "/export") { + assert.Contains(t, w.Body.String(), "readable transcript") + } else { + assert.Contains(t, w.Body.String(), "codex:portable") + } + assert.NotContains(t, w.Body.String(), "raw-row-") + } + pin() + w := request("/api/v1/pins") + require.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "remember this") + assert.Contains(t, w.Body.String(), "message_key") + split() + w = request("/api/v1/sessions/codex:portable") + require.Equal(t, http.StatusConflict, w.Code, w.Body.String()) + var conflict struct { + State string `json:"state"` + Variants []string `json:"variants"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &conflict)) + assert.Equal(t, "ambiguous", conflict.State) + require.Len(t, conflict.Variants, 2) + for _, v := range conflict.Variants { + assert.True(t, strings.HasPrefix(v, "codex:portable~")) + } + w = request("/api/v1/sessions/codex:portable/watch") + assert.Equal(t, http.StatusConflict, w.Code, w.Body.String()) +} + +func TestHostedWatchReportsConflictAndStops(t *testing.T) { + store, split, _ := postgres.HostedPublicFixture(t) + restore := sessionwatch.SetTimingsForTest(10*time.Millisecond, time.Second) + defer restore() + backend := service.NewReadOnlyBackend(store) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + events, err := backend.Watch(ctx, "codex:portable") + require.NoError(t, err) + split() + select { + case event, ok := <-events: + require.True(t, ok) + assert.Equal(t, "session.identity", event.Event) + var state db.SessionWatchState + require.NoError(t, json.Unmarshal([]byte(event.Data), &state)) + assert.Equal(t, "ambiguous", state.State) + require.Len(t, state.Variants, 2) + case <-ctx.Done(): + t.Fatal("watch did not report conflict") + } + select { + case _, ok := <-events: + assert.False(t, ok) + case <-ctx.Done(): + t.Fatal("watch did not close") + } +} diff --git a/internal/postgres/hosted_reports.go b/internal/postgres/hosted_reports.go new file mode 100644 index 0000000000..33ce95d6e8 --- /dev/null +++ b/internal/postgres/hosted_reports.go @@ -0,0 +1,289 @@ +package postgres + +import ( + "context" + "go.kenn.io/agentsview/internal/activity" + "go.kenn.io/agentsview/internal/db" + "slices" + "time" +) + +func hostedMapped[T any](ctx context.Context, h *HostedStore, read func() (T, error), collect func(*T, *hostedRefs)) (T, error) { + return hostedRead(ctx, h, func(_ hostedRevision) (T, error) { + v, err := read() + if err != nil { + return v, err + } + r := hostedRefs{} + collect(&v, &r) + err = r.mapIDs(ctx, h) + return v, err + }) +} +func (h *HostedStore) Search(ctx context.Context, f db.SearchFilter) (db.SearchPage, error) { + return hostedMapped(ctx, h, func() (db.SearchPage, error) { return h.physical.Search(ctx, f) }, func(p *db.SearchPage, r *hostedRefs) { + p.Results = slices.Clone(p.Results) + for i := range p.Results { + r.add(&p.Results[i].SessionID) + } + }) +} +func (h *HostedStore) SearchContent(ctx context.Context, f db.ContentSearchFilter) (db.ContentSearchPage, error) { + return hostedMapped(ctx, h, func() (db.ContentSearchPage, error) { + q := f + var err error + q.ExcludeSessionIDs, err = h.resolveBatch(ctx, f.ExcludeSessionIDs, true, false) + if err != nil { + return db.ContentSearchPage{}, err + } + return h.physical.SearchContent(ctx, q) + }, func(p *db.ContentSearchPage, r *hostedRefs) { + p.Matches = slices.Clone(p.Matches) + for i := range p.Matches { + m := &p.Matches[i] + r.add(&m.SessionID) + r.link(m.SessionID, &m.ParentSessionID) + m.ContextBefore = r.messages(m.ContextBefore) + m.ContextAfter = r.messages(m.ContextAfter) + } + }) +} +func (h *HostedStore) ListSecretFindings(ctx context.Context, f db.SecretFindingFilter) (db.SecretFindingPage, error) { + return hostedMapped(ctx, h, func() (db.SecretFindingPage, error) { return h.physical.ListSecretFindings(ctx, f) }, func(p *db.SecretFindingPage, r *hostedRefs) { + p.Findings = slices.Clone(p.Findings) + for i := range p.Findings { + r.add(&p.Findings[i].SessionID) + } + }) +} +func (h *HostedStore) SecretFindingSource(ctx context.Context, f db.SecretFinding) (string, bool, error) { + type result struct { + content string + ok bool + } + v, err := hostedDetail(ctx, h, f.SessionID, func(id string) (result, error) { + f.SessionID = id + c, ok, err := h.physical.SecretFindingSource(ctx, f) + return result{c, ok}, err + }) + return v.content, v.ok, err +} +func (h *HostedStore) GetSessionTiming(ctx context.Context, id string) (*db.SessionTiming, error) { + return hostedMapped(ctx, h, func() (*db.SessionTiming, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + session, err := h.physical.GetSession(ctx, target.SessionID) + if err != nil || session == nil { + return nil, err + } + turns, err := h.physical.queryTurnRows(ctx, target.SessionID) + if err != nil { + return nil, err + } + join := `LEFT JOIN LATERAL (SELECT min(target_session.started_at) AS started_at,max(target_session.ended_at) AS ended_at ` + hostedLinkFromSQL + ` AND owner.session_id=tc.session_id AND e.kind='call' AND e.ordinal=tc.message_ordinal AND e.call_index=tc.call_index HAVING count(DISTINCT target.session_id)=1) s_sub ON true` + if target.Legacy { + join = `LEFT JOIN sessions s_sub ON s_sub.id=tc.subagent_session_id` + } + calls, err := h.physical.queryCallRowsWithJoin(ctx, target.SessionID, join) + if err != nil { + return nil, err + } + return db.AssembleTiming(session, turns, calls, time.Now().UTC()), nil + }, func(p **db.SessionTiming, r *hostedRefs) { + if *p == nil { + return + } + v := **p + *p = &v + r.add(&v.SessionID) + if v.SlowestCall != nil { + c := *v.SlowestCall + v.SlowestCall = &c + r.parent(v.SessionID, &c.SubagentSessionID, nil) + } + v.Turns = slices.Clone(v.Turns) + for i := range v.Turns { + v.Turns[i].Calls = slices.Clone(v.Turns[i].Calls) + for j := range v.Turns[i].Calls { + r.parent(v.SessionID, &v.Turns[i].Calls[j].SubagentSessionID, nil) + } + } + }) +} +func (h *HostedStore) GetSessionUsage(ctx context.Context, id string, breakdown bool) (*db.SessionUsage, error) { + return hostedMapped(ctx, h, func() (*db.SessionUsage, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + return h.physical.GetSessionUsage(ctx, target.SessionID, breakdown) + }, func(p **db.SessionUsage, r *hostedRefs) { + if *p == nil { + return + } + v := **p + *p = &v + r.add(&v.SessionID) + v.Breakdown = slices.Clone(v.Breakdown) + for i := range v.Breakdown { + r.add(&v.Breakdown[i].SubagentSessionID) + } + }) +} +func (h *HostedStore) GetTopSessionsByCost(ctx context.Context, f db.UsageFilter, limit int) ([]db.TopSessionEntry, error) { + return hostedMapped(ctx, h, func() ([]db.TopSessionEntry, error) { return h.physical.GetTopSessionsByCost(ctx, f, limit) }, func(p *[]db.TopSessionEntry, r *hostedRefs) { + *p = slices.Clone(*p) + for i := range *p { + r.add(&(*p)[i].SessionID) + } + }) +} +func (h *HostedStore) GetAnalyticsTopSessions(ctx context.Context, f db.AnalyticsFilter, metric string) (db.TopSessionsResponse, error) { + return hostedMapped(ctx, h, func() (db.TopSessionsResponse, error) { return h.physical.GetAnalyticsTopSessions(ctx, f, metric) }, func(p *db.TopSessionsResponse, r *hostedRefs) { + p.Sessions = slices.Clone(p.Sessions) + for i := range p.Sessions { + r.add(&p.Sessions[i].ID) + } + }) +} +func (h *HostedStore) GetAnalyticsSignalSessions(ctx context.Context, f db.AnalyticsFilter, signal string, limit int) (db.SignalSessionsResponse, error) { + return hostedMapped(ctx, h, func() (db.SignalSessionsResponse, error) { + return h.physical.GetAnalyticsSignalSessions(ctx, f, signal, limit) + }, func(p *db.SignalSessionsResponse, r *hostedRefs) { + p.Sessions = slices.Clone(p.Sessions) + for i := range p.Sessions { + r.add(&p.Sessions[i].SessionID) + } + }) +} +func (h *HostedStore) RecentEdits(ctx context.Context, f db.RecentEditsParams) (db.RecentEditsResult, error) { + return hostedMapped(ctx, h, func() (db.RecentEditsResult, error) { return h.physical.RecentEdits(ctx, f) }, func(p *db.RecentEditsResult, r *hostedRefs) { + p.Files = slices.Clone(p.Files) + for i := range p.Files { + f := &p.Files[i] + r.add(&f.LastSessionID) + f.Edits = slices.Clone(f.Edits) + for j := range f.Edits { + r.add(&f.Edits[j].SessionID) + } + } + }) +} +func (h *HostedStore) ListArchiveWorktreeCandidates(ctx context.Context, f db.ArchiveWorktreeCandidateRequest) ([]db.WorktreeReclassificationCandidate, error) { + return hostedMapped(ctx, h, func() ([]db.WorktreeReclassificationCandidate, error) { + return h.physical.ListArchiveWorktreeCandidates(ctx, f) + }, func(p *[]db.WorktreeReclassificationCandidate, r *hostedRefs) { + *p = slices.Clone(*p) + for i := range *p { + c := &(*p)[i] + c.Examples = slices.Clone(c.Examples) + for j := range c.Examples { + r.add(&c.Examples[j].SessionID) + } + } + }) +} +func (r *hostedRefs) report(p *activity.Report) { + p.BySession = slices.Clone(p.BySession) + for i := range p.BySession { + r.add(&p.BySession[i].SessionID) + } + p.Intervals = slices.Clone(p.Intervals) + for i := range p.Intervals { + r.add(&p.Intervals[i].SessionID) + } +} +func (h *HostedStore) GetActivityReport(ctx context.Context, f db.AnalyticsFilter, q activity.Query) (activity.Report, error) { + return hostedMapped(ctx, h, func() (activity.Report, error) { return h.physical.GetActivityReport(ctx, f, q) }, func(p *activity.Report, r *hostedRefs) { r.report(p) }) +} + +// Map keys are copied into addressable entries for the same single batch lookup. +func hostedMapKeys[T any](m map[string]T, r *hostedRefs) func() map[string]T { + type entry struct { + key string + value T + } + entries := make([]entry, 0, len(m)) + for k, v := range m { + entries = append(entries, entry{k, v}) + } + for i := range entries { + r.add(&entries[i].key) + } + return func() map[string]T { + if m == nil { + return nil + } + out := make(map[string]T, len(m)) + for _, e := range entries { + out[e.key] = e.value + } + return out + } +} +func (h *HostedStore) BuildActivityReportArtifacts(ctx context.Context, f db.AnalyticsFilter, q activity.Query, progress activity.ProgressFunc) (activity.CandidateArtifacts, error) { + return hostedRead(ctx, h, func(_ hostedRevision) (activity.CandidateArtifacts, error) { + p, err := h.physical.BuildActivityReportArtifacts(ctx, f, q, progress) + if err != nil { + return p, err + } + r := hostedRefs{} + r.report(&p.Report) + p.Sessions = slices.Clone(p.Sessions) + for i := range p.Sessions { + r.add(&p.Sessions[i].SessionID) + } + membership := hostedMapKeys(p.Membership, &r) + if err = r.mapIDs(ctx, h); err != nil { + return p, err + } + p.Membership = membership() + return p, nil + }) +} +func (h *HostedStore) ActivityReportSourceProbe(ctx context.Context) (activity.SourceProbe, error) { + return hostedRead(ctx, h, func(rev hostedRevision) (activity.SourceProbe, error) { + p, err := h.physical.ActivityReportSourceProbe(ctx) + p.HostedIdentityRevision = rev.Identity + p.HostedSelectionRevision = rev.Selection + p.HostedCorpusRevision = rev.Corpus + return p, err + }) +} +func (h *HostedStore) GetSessionUsageRows(ctx context.Context, aliases []string) (*activity.SessionUsageRows, error) { + return hostedRead(ctx, h, func(_ hostedRevision) (*activity.SessionUsageRows, error) { + ids, err := h.resolveBatch(ctx, aliases, false, false) + if err != nil { + return nil, err + } + p, err := h.physical.GetSessionUsageRows(ctx, ids) + if err != nil || p == nil { + return p, err + } + v := *p + r := hostedRefs{} + v.Rows = slices.Clone(v.Rows) + for i := range v.Rows { + r.add(&v.Rows[i].SessionID) + r.add(&v.Rows[i].SourceSessionID) + } + raw := hostedMapKeys(v.RawOutputTokensBySession, &r) + dedup := hostedMapKeys(v.DeduplicatedOutputTokens, &r) + discarded := hostedMapKeys(v.DiscardedContributingSessions, &r) + coverage := hostedMapKeys(v.CanonicalTokenCoverageBySession, &r) + if err = r.mapIDs(ctx, h); err != nil { + return nil, err + } + v.RawOutputTokensBySession = raw() + v.DeduplicatedOutputTokens = dedup() + v.DiscardedContributingSessions = discarded() + v.CanonicalTokenCoverageBySession = coverage() + return &v, nil + }) +} + +var _ db.ActivityReportArtifactStore = (*HostedStore)(nil) +var _ db.ActivityReportProbeStore = (*HostedStore)(nil) +var _ db.ActivityReportTokenStore = (*HostedStore)(nil) diff --git a/internal/postgres/hosted_review_pgtest_test.go b/internal/postgres/hosted_review_pgtest_test.go new file mode 100644 index 0000000000..1de5323498 --- /dev/null +++ b/internal/postgres/hosted_review_pgtest_test.go @@ -0,0 +1,374 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +func waitHostedLocks(t *testing.T, f projectionFixture, ctx context.Context, count int) { + t.Helper() + require.Eventually(t, func() bool { + var n int + err := f.admin.QueryRowContext(ctx, `SELECT count(*) FROM pg_stat_activity WHERE usename=$1 AND wait_event_type='Lock'`, f.role).Scan(&n) + return err == nil && n >= count + }, 3*time.Second, 10*time.Millisecond) +} + +func TestHostedPinResponseUsesTransactionCohort(t *testing.T) { + f := newProjectionFixture(t) + first, accepted := f.accept(t, "device-a", "pin-race-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, first), first, projectionOutcome("before pin"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + old, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + next, _ := f.accept(t, "device-a", "pin-race-b", accepted.Receipt) + lease := f.lease(t, next) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + _, err = gate.ExecContext(ctx, `SELECT group_id FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, old.GroupID) + require.NoError(t, err) + published := make(chan error, 1) + go func() { published <- f.sink.Project(ctx, lease, next, projectionOutcome("after pin")) }() + waitHostedLocks(t, f, ctx, 1) + type response struct { + id int64 + err error + } + pinned := make(chan response, 1) + go func() { + id, err := h.PinMessage("codex:portable", 0, new("transaction pin")) + pinned <- response{id, err} + }() + waitHostedLocks(t, f, ctx, 2) + require.NoError(t, gate.Commit()) + require.NoError(t, <-published) + result := <-pinned + require.NoError(t, result.err) + require.Positive(t, result.id) + var content, note string + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT m.content,p.note FROM pinned_messages p JOIN messages m ON m.session_id=p.session_id AND m.ordinal=p.ordinal WHERE p.id=$1`, result.id).Scan(&content, ¬e)) + assert.Equal(t, "after pin", content) + assert.Equal(t, "transaction pin", note) +} + +func TestHostedLegacyWriteFencesFirstRawPublication(t *testing.T) { + f := newProjectionFixture(t) + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('codex:portable','archive','machine','codex')`) + require.NoError(t, err) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + manifest, _ := f.accept(t, "device-a", "first-raw-publication", "") + lease := f.lease(t, manifest) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + require.NoError(t, lockHostedAliases(ctx, gate, []string{"codex:portable"})) + published := make(chan error, 1) + go func() { published <- f.sink.Project(ctx, lease, manifest, projectionOutcome("raw collision")) }() + waitHostedLocks(t, f, ctx, 1) + starred := make(chan error, 1) + go func() { _, err := h.StarSession("codex:portable"); starred <- err }() + waitHostedLocks(t, f, ctx, 2) + require.NoError(t, gate.Commit()) + require.NoError(t, <-published) + var ambiguity *db.SessionIdentityError + require.ErrorAs(t, <-starred, &ambiguity) + var star bool + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM starred_sessions WHERE session_id='codex:portable')`).Scan(&star)) + assert.False(t, star) + explicit := legacyPublicVariant("codex:portable") + ok, err := h.StarSession(explicit) + require.NoError(t, err) + assert.True(t, ok) + require.NoError(t, h.RenameSession(explicit, new("legacy title"))) + require.NoError(t, h.SoftDeleteSession(explicit)) + n, err := h.RestoreSession(explicit) + require.NoError(t, err) + assert.EqualValues(t, 1, n) + require.NoError(t, h.UnstarSession(explicit)) + + _, err = f.runtime.ExecContext(ctx, `INSERT INTO messages(session_id,ordinal,role,content) VALUES('codex:portable',0,'user','legacy pinned text')`) + require.NoError(t, err) + pinID, err := h.PinMessage(explicit, 0, new("legacy note")) + require.NoError(t, err) + require.Positive(t, pinID) + require.NoError(t, h.UnpinMessage(explicit, 0)) + legacy, err := h.GetSession(ctx, explicit) + require.NoError(t, err) + require.NotNil(t, legacy) + assert.Equal(t, "legacy title", *legacy.DisplayName) + + require.NoError(t, h.SoftDeleteSession(explicit)) + removed, err := h.DeleteSessionIfTrashed(explicit) + require.NoError(t, err) + assert.EqualValues(t, 1, removed) + raw, err := h.GetSession(ctx, "codex:portable") + require.NoError(t, err) + require.NotNil(t, raw) + assert.Equal(t, "codex:portable", raw.ID) +} + +func TestHostedLegacyIdentityTriggerContentionIsAtomic(t *testing.T) { + f := newProjectionFixture(t) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + before, err := h.revision(t.Context()) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + require.NoError(t, lockHostedAliases(ctx, gate, []string{"codex:portable"})) + _, err = f.admin.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent,tenant_id) VALUES('codex:portable','archive','machine','codex',$1)`, f.tenant) + var pgerr *pgconn.PgError + require.ErrorAs(t, err, &pgerr) + assert.Equal(t, "40001", pgerr.Code) + after, err := h.revision(ctx) + require.NoError(t, err) + assert.Equal(t, before, after) + var exists bool + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sessions WHERE id='codex:portable')`).Scan(&exists)) + assert.False(t, exists) + require.NoError(t, gate.Commit()) + _, err = f.admin.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent,tenant_id) VALUES('codex:portable','archive','machine','codex',$1)`, f.tenant) + require.NoError(t, err) + after, err = h.revision(ctx) + require.NoError(t, err) + assert.Equal(t, before.Identity+1, after.Identity) +} + +func TestHostedLegacyTrashPassCannotDeleteNewRawTrash(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "late-trash", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("late raw trash"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + n, err := h.core.EmptyTrash(t.Context()) + require.NoError(t, err) + assert.Zero(t, n) + require.NoError(t, h.SoftDeleteSession("codex:portable")) + n, err = h.physical.emptyTrash(true) + require.NoError(t, err) + assert.Zero(t, n) + var cause sql.NullString + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT deletion_cause FROM sessions WHERE provenance_kind='raw'`).Scan(&cause)) + assert.Equal(t, "user", cause.String) + n, err = h.EmptyTrash() + require.NoError(t, err) + assert.Equal(t, 1, n) +} + +func TestHostedPinInventorySkipsFalseHistoricalPayloads(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "unpinned-history", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("historical pin"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = h.PinMessage("codex:portable", 0, nil) + require.NoError(t, err) + require.NoError(t, h.UnpinMessage("codex:portable", 0)) + // Corruption is deliberate proof that an ineffective historical candidate is + // never loaded or decoded by an empty inventory request. + _, err = f.runtime.ExecContext(t.Context(), `UPDATE raw_content_revisions SET payload='not-json'::bytea`) + require.NoError(t, err) + pins, err := h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + assert.Empty(t, pins) +} + +func TestHostedPinPageBoundsHydrationAndOrdersUnresolved(t *testing.T) { + f := newProjectionFixture(t) + first, accepted := f.accept(t, "device-a", "bounded-pin-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, first), first, projectionOutcome("durable old pin"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = h.PinMessage("codex:portable", 0, new("newest retained pin")) + require.NoError(t, err) + next, _ := f.accept(t, "device-a", "bounded-pin-b", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("replacement"))) + _, err = f.runtime.ExecContext(t.Context(), `UPDATE raw_pins SET created_at='2025-01-01'`) + require.NoError(t, err) + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('legacy-pin-page','archive','machine','codex'); INSERT INTO messages(session_id,ordinal,role,content) SELECT 'legacy-pin-page',n,'user','legacy pin '||n FROM generate_series(1,500)n; INSERT INTO pinned_messages(session_id,message_id,ordinal,created_at) SELECT 'legacy-pin-page',n,n,'2026-01-01'::timestamptz+n*interval '1 second' FROM generate_series(1,500)n`) + require.NoError(t, err) + target, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + var payload []byte + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT payload FROM raw_content_revisions WHERE session_id=$1`, target.SessionID).Scan(&payload)) + _, err = f.runtime.ExecContext(t.Context(), `UPDATE raw_content_revisions SET payload='not-json'::bytea WHERE session_id=$1`, target.SessionID) + require.NoError(t, err) + pins, err := h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + require.Len(t, pins, 500) + assert.Equal(t, "legacy pin 500", *pins[0].Content) + assert.Equal(t, "legacy pin 1", *pins[499].Content) + _, err = f.runtime.ExecContext(t.Context(), `UPDATE raw_content_revisions SET payload=$2 WHERE session_id=$1`, target.SessionID, payload) + require.NoError(t, err) + _, err = f.runtime.ExecContext(t.Context(), `UPDATE raw_pins SET created_at='2027-01-01'`) + require.NoError(t, err) + pins, err = h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + require.Len(t, pins, 500) + assert.Equal(t, "codex:portable", pins[0].SessionID) + assert.True(t, pins[0].Unresolved) + assert.Equal(t, "2027-01-01T00:00:00Z", pins[0].CreatedAt) + assert.Equal(t, "newest retained pin", *pins[0].Note) + assert.Equal(t, "legacy pin 500", *pins[1].Content) + assert.Equal(t, "legacy pin 2", *pins[499].Content) +} + +func TestHostedLegacyIdentityTriggerDoesNotWaitBehindGroup(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "group-contention", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("existing raw"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + target, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + before, err := h.revision(t.Context()) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + _, err = gate.ExecContext(ctx, `SELECT group_id FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, target.GroupID) + require.NoError(t, err) + _, err = f.admin.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent,tenant_id) VALUES('codex:portable','archive','machine','codex',$1)`, f.tenant) + var pgerr *pgconn.PgError + require.ErrorAs(t, err, &pgerr) + assert.Equal(t, "40001", pgerr.Code) + after, err := h.revision(ctx) + require.NoError(t, err) + assert.Equal(t, before, after) + require.NoError(t, gate.Commit()) + _, err = f.admin.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent,tenant_id) VALUES('codex:portable','archive','machine','codex',$1)`, f.tenant) + require.NoError(t, err) + _, err = h.StarSession("codex:portable") + var ambiguity *db.SessionIdentityError + require.ErrorAs(t, err, &ambiguity) +} + +func TestHostedPinPageAppliesEffectiveCohortOverrides(t *testing.T) { + f := newProjectionFixture(t) + a, _ := f.accept(t, "device-a", "pin-cohort-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("cohort pin"))) + b, _ := f.accept(t, "device-b", "pin-cohort-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("cohort pin"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = h.PinMessage("codex:portable", 0, new("inherited")) + require.NoError(t, err) + // Retained member overrides can differ after reconvergence. A false override + // on A must not suppress B's inherited group pin. + aa := f.alias(t, a) + ba := f.alias(t, b) + // Target the durable per-member overrides directly, then materialize using + // the normal curation path; this isolates heterogeneous membership history. + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO raw_pins(group_id,branch_id,message_key,ordinal,content_revision,pinned,note,created_at) SELECT p.group_id,a.anchor_branch,p.message_key,p.ordinal,p.content_revision,false,'suppressed',p.created_at FROM raw_pins p JOIN raw_session_public_aliases a ON a.group_id=p.group_id WHERE p.branch_id='' AND a.alias_id=$1`, aa) + require.NoError(t, err) + require.NoError(t, h.RenameSession("codex:portable", nil)) + pins, err := h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, "inherited", *pins[0].Note) + assert.False(t, pins[0].Unresolved) + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO raw_pins(group_id,branch_id,message_key,ordinal,content_revision,pinned,note,created_at) SELECT p.group_id,a.anchor_branch,p.message_key,p.ordinal,p.content_revision,false,'suppressed',p.created_at FROM raw_pins p JOIN raw_session_public_aliases a ON a.group_id=p.group_id WHERE p.branch_id='' AND a.alias_id=$1`, ba) + require.NoError(t, err) + require.NoError(t, h.RenameSession("codex:portable", nil)) + pins, err = h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + assert.Empty(t, pins) +} + +func TestHostedAliasBucketsBoundContentionWithoutGlobalLock(t *testing.T) { + f := newProjectionFixture(t) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + var same, different string + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT 'legacy-bucket-'||n FROM generate_series(1,10000)n WHERE (hashtextextended('legacy-bucket-'||n,0)&255)=(hashtextextended('codex:portable',0)&255) LIMIT 1`).Scan(&same)) + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT 'legacy-bucket-'||n FROM generate_series(1,10000)n WHERE (hashtextextended('legacy-bucket-'||n,0)&255)<>(hashtextextended('codex:portable',0)&255) AND (hashtextextended(hosted_legacy_alias('legacy-bucket-'||n),0)&255)<>(hashtextextended('codex:portable',0)&255) LIMIT 1`).Scan(&different)) + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + require.NoError(t, lockHostedAliases(ctx, gate, []string{"codex:portable"})) + _, err = f.runtime.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent) VALUES($1,'archive','machine','codex')`, same) + var pgerr *pgconn.PgError + require.ErrorAs(t, err, &pgerr) + assert.Equal(t, "40001", pgerr.Code) + _, err = f.runtime.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent) VALUES($1,'independent','machine','codex')`, different) + require.NoError(t, err) + var project string + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT project FROM sessions WHERE id=$1`, different).Scan(&project)) + assert.Equal(t, "independent", project) + require.NoError(t, gate.Commit()) + _, err = f.runtime.ExecContext(ctx, `INSERT INTO sessions(id,project,machine,agent) VALUES($1,'retried','machine','codex')`, same) + require.NoError(t, err) + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT project FROM sessions WHERE id=$1`, same).Scan(&project)) + assert.Equal(t, "retried", project) +} + +func TestHostedLegacyWriteFencesFirstVibeFallbackPublication(t *testing.T) { + f := newProjectionFixture(t) + manifest, _ := f.accept(t, "device-a", "first-vibe-fallback", "", parser.AgentVibe) + const canonical = "vibe:portable" + group, _ := rawGroupID(manifest, db.Session{ID: canonical, Agent: "vibe", SourceSessionID: "portable"}) + branch := canonical + "~" + rawDigest("branch-v1", group, rawSourceID(manifest)) + var directory string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT 'session_fallback_'||n FROM generate_series(1,10000)n WHERE (hashtextextended('vibe:session_fallback_'||n,0)&255) NOT IN ((hashtextextended($1,0)&255),(hashtextextended($2,0)&255)) LIMIT 1`, canonical, branch).Scan(&directory)) + fallback := "vibe:" + directory + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES($1,'archive','machine','vibe')`, fallback) + require.NoError(t, err) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + outcome := projectionOutcome("published Vibe transcript") + outcome.Outcome.Results[0].Result.Session.Agent = parser.AgentVibe + outcome.Outcome.Results[0].Result.Session.ID = canonical + outcome.Outcome.Results[0].Result.Session.File.Path = "logs/" + directory + "/messages.jsonl" + lease := f.lease(t, manifest) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + require.NoError(t, lockHostedAliases(ctx, gate, []string{fallback})) + published := make(chan error, 1) + go func() { published <- f.sink.Project(ctx, lease, manifest, outcome) }() + // The fallback bucket differs from both pre-existing lock candidates. Without + // its explicit inclusion, first publication passes this gate entirely. + waitHostedLocks(t, f, ctx, 1) + starred := make(chan error, 1) + go func() { _, err := h.StarSession(fallback); starred <- err }() + waitHostedLocks(t, f, ctx, 2) + require.NoError(t, gate.Commit()) + require.NoError(t, <-published) + var ambiguity *db.SessionIdentityError + require.ErrorAs(t, <-starred, &ambiguity) + assert.Contains(t, ambiguity.Variants, legacyPublicVariant(fallback)) + var star bool + require.NoError(t, f.runtime.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM starred_sessions WHERE session_id=$1)`, fallback).Scan(&star)) + assert.False(t, star) + messages, err := h.GetAllMessages(ctx, canonical) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "published Vibe transcript", messages[0].Content) + ok, err := h.StarSession(legacyPublicVariant(fallback)) + require.NoError(t, err) + assert.True(t, ok) +} diff --git a/internal/postgres/hosted_session_ids.go b/internal/postgres/hosted_session_ids.go new file mode 100644 index 0000000000..d1bb830652 --- /dev/null +++ b/internal/postgres/hosted_session_ids.go @@ -0,0 +1,269 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "go.kenn.io/agentsview/internal/db" + "slices" +) + +// The legacy variant uses a separate namespace and never exposes a raw physical +// row as lookup authority. A retained archive row has no raw rebuild provenance. +const hostedLegacyAliasSQL = `hosted_legacy_alias(id)` + +type hostedIdentity struct { + RawIdentity + Legacy bool +} + +func (h *HostedStore) resolve(ctx context.Context, alias string) (hostedIdentity, error) { + raw, err := h.core.Resolve(ctx, alias) + if err != nil { + return hostedIdentity{}, err + } + var legacy string + err = h.physical.pg.QueryRowContext(ctx, `SELECT id FROM sessions WHERE provenance_kind='legacy' AND (id=$1 OR `+hostedLegacyAliasSQL+`=$1)`, alias).Scan(&legacy) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return hostedIdentity{}, err + } + if legacy != "" { + if raw.State != RawIdentityGone { + var variant string + err = h.physical.pg.QueryRowContext(ctx, `SELECT `+hostedLegacyAliasSQL+` FROM sessions WHERE id=$1`, legacy).Scan(&variant) + if err != nil { + return hostedIdentity{}, err + } + return hostedIdentity{}, &db.SessionIdentityError{State: "ambiguous", Variants: append(raw.Variants, variant)} + } + canonical := legacy + refs := hostedRefs{} + refs.add(&canonical) + if err = refs.mapIDs(ctx, h); err != nil { + return hostedIdentity{}, err + } + return hostedIdentity{State: RawIdentityUnique, SessionID: legacy, PublicID: canonical, Legacy: true}, nil + } + if raw.State == RawIdentityAmbiguous { + return hostedIdentity{}, &db.SessionIdentityError{State: "ambiguous", Variants: raw.Variants} + } + if raw.State == RawIdentityUnique { + canonical := raw.SessionID + r := hostedRefs{} + r.add(&canonical) + if err = r.mapIDs(ctx, h); err != nil { + return hostedIdentity{}, err + } + raw.PublicID = canonical + } + return hostedIdentity{RawIdentity: raw}, nil +} + +// hostedRefs collects only declared relational fields. Pointers always belong to +// a result-owned copy; raw JSON, provider IDs and source text never enter here. +type hostedLink struct { + owner, alias string + field *string + optional **string + plural *[]string +} +type hostedRefs struct { + fields []*string + links []hostedLink +} + +func (r *hostedRefs) link(owner string, p *string) { + if p != nil && *p != "" { + r.links = append(r.links, hostedLink{owner: owner, alias: *p, field: p}) + } +} +func (r *hostedRefs) parent(owner string, p **string, plural *[]string) { + if *p != nil && **p != "" { + r.links = append(r.links, hostedLink{owner: owner, alias: **p, optional: p, plural: plural}) + } +} + +func (r *hostedRefs) add(p *string) { + if p != nil && *p != "" { + r.fields = append(r.fields, p) + } +} +func (r *hostedRefs) mapIDs(ctx context.Context, h *HostedStore) error { + edges := map[string][]string{} + owners := make([]string, 0, len(r.links)) + for _, l := range r.links { + owners = append(owners, l.owner) + } + slices.Sort(owners) + owners = slices.Compact(owners) + if len(owners) > 0 { + rows, err := h.physical.pg.QueryContext(ctx, `SELECT DISTINCT owner.session_id,e.target_alias,target.session_id `+hostedLinkFromSQL+` AND owner.session_id=ANY($1)`, owners) + if err != nil { + return err + } + for rows.Next() { + var owner, alias, target string + if err = rows.Scan(&owner, &alias, &target); err != nil { + rows.Close() + return err + } + key := owner + "\x00" + alias + edges[key] = append(edges[key], target) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + } + ids := make([]string, 0, len(r.fields)) + for _, p := range r.fields { + ids = append(ids, *p) + } + for _, l := range r.links { + ids = append(ids, l.owner, l.alias) + ids = append(ids, edges[l.owner+"\x00"+l.alias]...) + } + slices.Sort(ids) + ids = slices.Compact(ids) + if len(ids) == 0 { + return nil + } + rows, err := h.physical.pg.QueryContext(ctx, ` + WITH targets AS (SELECT id,provenance_kind,raw_group_id FROM sessions WHERE id=ANY($1)), + groups AS (SELECT DISTINCT raw_group_id AS group_id FROM targets WHERE raw_group_id<>''), + cohorts AS (SELECT s.raw_group_id AS group_id,count(*) AS n FROM sessions s JOIN groups g ON g.group_id=s.raw_group_id GROUP BY s.raw_group_id), + representatives AS (SELECT b.session_id,min(b.branch_id) AS branch FROM raw_session_branches b JOIN groups g ON g.group_id=b.group_id WHERE b.active AND NOT COALESCE((SELECT c.value::boolean FROM raw_curation c WHERE c.group_id=b.group_id AND c.field='excluded' AND c.branch_id IN ('',b.branch_id) ORDER BY c.branch_id DESC LIMIT 1),false) GROUP BY b.session_id) + SELECT t.id,t.provenance_kind, CASE WHEN t.provenance_kind='legacy' THEN + CASE WHEN EXISTS(SELECT 1 FROM raw_session_groups g JOIN sessions s ON s.raw_group_id=g.group_id WHERE g.base_alias=t.id) THEN hosted_legacy_alias(t.id) ELSE t.id END + ELSE CASE WHEN c.n>1 OR EXISTS(SELECT 1 FROM sessions l WHERE l.id=g.base_alias AND l.provenance_kind='legacy') OR (SELECT count(DISTINCT a.group_id) FROM raw_session_public_aliases a JOIN sessions s ON s.raw_group_id=a.group_id WHERE a.alias_id=g.base_alias)>1 THEN g.base_alias||'~'||r.branch ELSE g.base_alias END END + FROM targets t LEFT JOIN raw_session_groups g ON g.group_id=t.raw_group_id LEFT JOIN cohorts c ON c.group_id=g.group_id LEFT JOIN representatives r ON r.session_id=t.id`, ids) + if err != nil { + return err + } + defer rows.Close() + mapped := map[string]string{} + kinds := map[string]string{} + for rows.Next() { + var id, kind string + var alias sql.NullString + if err = rows.Scan(&id, &kind, &alias); err != nil { + return err + } + mapped[id] = alias.String + kinds[id] = kind + } + if err = rows.Err(); err != nil { + return err + } + for _, p := range r.fields { + *p = mapped[*p] + } + for _, l := range r.links { + targets := edges[l.owner+"\x00"+l.alias] + if kinds[l.owner] == "legacy" { + targets = []string{l.alias} + } + variants := make([]string, 0, len(targets)) + for _, id := range targets { + if mapped[id] != "" { + variants = append(variants, mapped[id]) + } + } + slices.Sort(variants) + variants = slices.Compact(variants) + if l.plural != nil { + *l.plural = nil + if len(variants) > 1 { + *l.plural = variants + } + } + if len(variants) > 1 && l.plural == nil { + return &db.SessionIdentityError{State: "ambiguous", Variants: variants} + } + value := "" + if len(variants) == 1 { + value = variants[0] + } + if l.field != nil { + *l.field = value + } + if l.optional != nil { + *l.optional = nil + if value != "" { + *l.optional = &value + } + } + } + return nil +} +func (r *hostedRefs) session(s *db.Session) { + r.add(&s.ID) + r.parent(s.ID, &s.ParentSessionID, &s.ParentSessionIDs) + s.ParserParentSessionID = nil +} +func (r *hostedRefs) messages(msgs []db.Message) []db.Message { + msgs = slices.Clone(msgs) + for i := range msgs { + m := &msgs[i] + r.add(&m.SessionID) + m.ToolCalls = slices.Clone(m.ToolCalls) + for j := range m.ToolCalls { + c := &m.ToolCalls[j] + r.add(&c.SessionID) + r.link(m.SessionID, &c.SubagentSessionID) + c.ResultEvents = slices.Clone(c.ResultEvents) + for k := range c.ResultEvents { + r.link(m.SessionID, &c.ResultEvents[k].SubagentSessionID) + } + } + } + return msgs +} + +// resolveBatch resolves collection inputs in one query. Singular curation keeps +// the richer core resolver and its transaction/group-lock scope. +func (h *HostedStore) resolveBatch(ctx context.Context, aliases []string, exclude, strict bool) ([]string, error) { + if len(aliases) == 0 { + return nil, nil + } + rows, err := h.physical.pg.QueryContext(ctx, `SELECT DISTINCT a.alias_id,b.session_id FROM raw_session_public_aliases a JOIN raw_session_branches b ON b.group_id=a.group_id AND b.active AND (a.anchor_branch='' OR a.anchor_branch=b.branch_id) JOIN sessions s ON s.id=b.session_id WHERE a.alias_id=ANY($1) AND NOT COALESCE((SELECT c.value::boolean FROM raw_curation c WHERE c.group_id=b.group_id AND c.field='excluded' AND c.branch_id IN ('',b.branch_id) ORDER BY c.branch_id DESC LIMIT 1),false) + UNION SELECT requested.alias,id FROM unnest($1::text[]) requested(alias) JOIN sessions s ON (s.id=requested.alias OR hosted_legacy_alias(s.id)=requested.alias) WHERE s.provenance_kind='legacy'`, aliases) + if err != nil { + return nil, err + } + targets := map[string][]string{} + for rows.Next() { + var alias, id string + if err = rows.Scan(&alias, &id); err != nil { + rows.Close() + return nil, err + } + targets[alias] = append(targets[alias], id) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + var ids []string + for _, alias := range aliases { + matches := targets[alias] + if len(matches) == 0 && strict { + return nil, &db.SessionIdentityError{State: "gone"} + } + if len(matches) > 1 && !exclude { + r := hostedRefs{} + for i := range matches { + r.add(&matches[i]) + } + if err = r.mapIDs(ctx, h); err != nil { + return nil, err + } + slices.Sort(matches) + return nil, &db.SessionIdentityError{State: "ambiguous", Variants: matches} + } + ids = append(ids, matches...) + } + return ids, nil +} diff --git a/internal/postgres/hosted_sessions.go b/internal/postgres/hosted_sessions.go new file mode 100644 index 0000000000..90775c86b2 --- /dev/null +++ b/internal/postgres/hosted_sessions.go @@ -0,0 +1,234 @@ +package postgres + +import ( + "context" + "errors" + "go.kenn.io/agentsview/internal/db" + "slices" + "strings" +) + +func (h *HostedStore) GetSession(ctx context.Context, id string) (*db.Session, error) { + return h.getSession(ctx, id, false) +} +func (h *HostedStore) GetSessionFull(ctx context.Context, id string) (*db.Session, error) { + return h.getSession(ctx, id, true) +} +func (h *HostedStore) getSession(ctx context.Context, id string, full bool) (*db.Session, error) { + return hostedRead(ctx, h, func(_ hostedRevision) (*db.Session, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + var s *db.Session + if full { + s, err = h.physical.GetSessionFull(ctx, target.SessionID) + } else { + s, err = h.physical.GetSession(ctx, target.SessionID) + } + if err != nil || s == nil { + return s, err + } + out := *s + r := hostedRefs{} + r.session(&out) + err = r.mapIDs(ctx, h) + return &out, err + }) +} +func (h *HostedStore) ListSessions(ctx context.Context, f db.SessionFilter) (db.SessionPage, error) { + return hostedRead(ctx, h, func(rev hostedRevision) (db.SessionPage, error) { + q := f + var err error + q.Cursor, err = h.openCursor(f.Cursor, rev) + if err != nil { + return db.SessionPage{}, err + } + p, err := h.physical.ListSessions(ctx, q) + if err != nil { + return p, err + } + p.Sessions = slices.Clone(p.Sessions) + r := hostedRefs{} + for i := range p.Sessions { + r.session(&p.Sessions[i]) + } + if err = r.mapIDs(ctx, h); err != nil { + return p, err + } + p.NextCursor, err = h.sealCursor(p.NextCursor, rev) + return p, err + }) +} +func (h *HostedStore) GetSidebarSessionIndex(ctx context.Context, f db.SessionFilter) (db.SidebarSessionIndex, error) { + return hostedRead(ctx, h, func(rev hostedRevision) (db.SidebarSessionIndex, error) { + q := f + var err error + q.Cursor, err = h.openCursor(f.Cursor, rev) + if err != nil { + return db.SidebarSessionIndex{}, err + } + p, err := h.physical.GetSidebarSessionIndex(ctx, q) + if err != nil { + return p, err + } + p.Sessions = slices.Clone(p.Sessions) + r := hostedRefs{} + for i := range p.Sessions { + r.add(&p.Sessions[i].ID) + r.parent(p.Sessions[i].ID, &p.Sessions[i].ParentSessionID, &p.Sessions[i].ParentSessionIDs) + } + if err = r.mapIDs(ctx, h); err != nil { + return p, err + } + p.NextCursor, err = h.sealCursor(p.NextCursor, rev) + return p, err + }) +} +func (h *HostedStore) GetChildSessions(ctx context.Context, id string) ([]db.Session, error) { + return hostedRead(ctx, h, func(_ hostedRevision) ([]db.Session, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + var children []db.Session + if target.Legacy { + children, err = h.physical.GetChildSessions(ctx, target.SessionID) + } else { + rows, e := h.physical.pg.QueryContext(ctx, `SELECT `+pgSessionCols+` FROM sessions WHERE id IN (SELECT owner.session_id `+hostedLinkFromSQL+` AND e.kind='parent' AND target.session_id=$1) AND deleted_at IS NULL ORDER BY COALESCE(started_at,created_at),id`, target.SessionID) + if e != nil { + return nil, e + } + children, err = scanPGSessionRows(rows) + rows.Close() + } + s := children + if err != nil { + return nil, err + } + s = slices.Clone(s) + r := hostedRefs{} + for i := range s { + s[i].ParentSessionID = nil + r.session(&s[i]) + } + err = r.mapIDs(ctx, h) + for i := range s { + parent := target.PublicID + s[i].ParentSessionID = &parent + } + return s, err + }) +} +func (h *HostedStore) FindSessionIDsByPartial(ctx context.Context, partial string, limit int) ([]string, error) { + return h.findAliases(ctx, partial, limit, false) +} +func (h *HostedStore) FindSessionIDsByRawSuffix(ctx context.Context, suffix string, limit int) ([]string, error) { + return h.findAliases(ctx, suffix, limit, true) +} +func (h *HostedStore) findAliases(ctx context.Context, term string, limit int, suffix bool) ([]string, error) { + if limit <= 0 || limit > db.MaxSessionLimit { + limit = db.MaxSessionLimit + } + return hostedRead(ctx, h, func(_ hostedRevision) ([]string, error) { + predicate := `strpos(a.alias_id,$1)>0` + if suffix { + predicate = `right(a.alias_id,length($1))=$1` + } + rows, err := h.physical.pg.QueryContext(ctx, `SELECT DISTINCT s.id FROM raw_session_public_aliases a JOIN raw_session_branches b ON b.group_id=a.group_id AND (a.anchor_branch='' OR a.anchor_branch=b.branch_id) JOIN sessions s ON s.id=b.session_id WHERE b.active AND NOT COALESCE((SELECT c.value::boolean FROM raw_curation c WHERE c.group_id=b.group_id AND c.field='excluded' AND c.branch_id IN ('',b.branch_id) ORDER BY c.branch_id DESC LIMIT 1),false) AND `+predicate+` UNION SELECT id FROM sessions WHERE provenance_kind='legacy' AND `+func() string { + if suffix { + return `right(id,length($1))=$1` + } + return `strpos(id,$1)>0` + }()+` ORDER BY 1 LIMIT $2`, term, limit) + if err != nil { + return nil, err + } + var ids []string + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + rows.Close() + return nil, err + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + r := hostedRefs{} + for i := range ids { + r.add(&ids[i]) + } + err = r.mapIDs(ctx, h) + slices.Sort(ids) + return slices.Compact(ids), err + }) +} +func (h *HostedStore) GetMessages(ctx context.Context, id string, from, limit int, asc bool) ([]db.Message, error) { + return h.readMessages(ctx, id, func(p string) ([]db.Message, error) { return h.physical.GetMessages(ctx, p, from, limit, asc) }) +} +func (h *HostedStore) GetMessagesWindow(ctx context.Context, id string, w db.MessageWindow) ([]db.Message, error) { + return h.readMessages(ctx, id, func(p string) ([]db.Message, error) { return h.physical.GetMessagesWindow(ctx, p, w) }) +} +func (h *HostedStore) GetAllMessages(ctx context.Context, id string) ([]db.Message, error) { + return h.readMessages(ctx, id, func(p string) ([]db.Message, error) { return h.physical.GetAllMessages(ctx, p) }) +} +func (h *HostedStore) readMessages(ctx context.Context, id string, read func(string) ([]db.Message, error)) ([]db.Message, error) { + return hostedRead(ctx, h, func(_ hostedRevision) ([]db.Message, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + return nil, err + } + msgs, err := read(target.SessionID) + if err != nil { + return nil, err + } + r := hostedRefs{} + msgs = r.messages(msgs) + err = r.mapIDs(ctx, h) + return msgs, err + }) +} +func hostedDetail[T any](ctx context.Context, h *HostedStore, id string, read func(string) (T, error)) (T, error) { + return hostedRead(ctx, h, func(_ hostedRevision) (T, error) { + target, err := h.resolve(ctx, id) + if err != nil || target.SessionID == "" { + var zero T + return zero, err + } + return read(target.SessionID) + }) +} +func (h *HostedStore) GetResumeModelCounts(ctx context.Context, id string) ([]db.ModelCount, error) { + return hostedDetail(ctx, h, id, func(p string) ([]db.ModelCount, error) { return h.physical.GetResumeModelCounts(ctx, p) }) +} +func (h *HostedStore) GetSessionActivity(ctx context.Context, id string) (*db.SessionActivityResponse, error) { + return hostedDetail(ctx, h, id, func(p string) (*db.SessionActivityResponse, error) { return h.physical.GetSessionActivity(ctx, p) }) +} +func (h *HostedStore) SearchSession(ctx context.Context, id, query string) ([]int, error) { + return hostedDetail(ctx, h, id, func(p string) ([]int, error) { return h.physical.SearchSession(ctx, p, query) }) +} + +func (h *HostedStore) GetProviderResumeID(ctx context.Context, id string) (string, error) { + session, err := h.GetSessionFull(ctx, id) + if err != nil { + return "", err + } + if session == nil { + return "", &db.SessionIdentityError{State: "gone"} + } + if session.SourceSessionID != "" { + return session.SourceSessionID, nil + } + target, err := h.resolve(ctx, id) + if err != nil { + return "", err + } + if target.Legacy { + return strings.TrimPrefix(target.SessionID, session.Agent+":"), nil + } + return "", errors.New("provider resume identity is unavailable") +} diff --git a/internal/postgres/hosted_store.go b/internal/postgres/hosted_store.go new file mode 100644 index 0000000000..4b94812e52 --- /dev/null +++ b/internal/postgres/hosted_store.go @@ -0,0 +1,224 @@ +package postgres + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/base64" + "errors" + "fmt" + "sync" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" +) + +// HostedStore is the explicit public identity boundary around physical queries. +// It deliberately does not embed Store: new capabilities require a mapping choice. +type HostedStore struct { + physical *Store + core *RawProjectionStore + tenant string + cursorMu sync.RWMutex + cursorKey [32]byte +} + +var _ db.Store = (*HostedStore)(nil) + +type hostedRevision struct{ Identity, Selection, Corpus int64 } + +var ErrHostedIdentityChanged = errors.New("hosted identity changed during read; retry the request") +var ErrHostedCursor = errors.New("invalid or expired hosted cursor") + +func newHostedAdapter(pg *sql.DB, tenant string) (*HostedStore, error) { + var revisionTrigger bool + if err := pg.QueryRowContext(context.Background(), `SELECT EXISTS( + SELECT 1 FROM pg_trigger t JOIN pg_proc p ON p.oid=t.tgfoid JOIN pg_language l ON l.oid=p.prolang + WHERE t.tgrelid='sessions'::regclass AND t.tgname='hosted_legacy_revision' + AND t.tgenabled='O' AND t.tgtype=29 AND t.tgqual IS NULL AND t.tgnargs=0 + AND t.tgattr=''::int2vector AND NOT t.tgisinternal AND t.tgconstraint=0 + AND NOT t.tgdeferrable AND NOT t.tginitdeferred + AND p.oid='hosted_legacy_revision()'::regprocedure + AND p.pronamespace=(SELECT relnamespace FROM pg_class WHERE oid='sessions'::regclass) + AND NOT p.prosecdef AND p.proconfig IS NULL AND p.prosrc=$1 AND p.provolatile='v' + AND l.lanname='plpgsql' +) AND EXISTS( + SELECT 1 FROM pg_proc p JOIN pg_language l ON l.oid=p.prolang + WHERE p.oid='hosted_legacy_alias(text)'::regprocedure AND NOT p.prosecdef + AND p.proconfig IS NULL AND p.prosrc=$2 AND p.provolatile='i' AND p.proisstrict + AND l.lanname='sql' +)`, hostedLegacyRevisionBody, hostedLegacyAliasBody).Scan(&revisionTrigger); err != nil { + return nil, err + } + if !revisionTrigger { + return nil, errors.New("hosted identity revision protection is missing or altered; owner must reprovision the schema") + } + core, err := NewRawProjectionStore(pg, RawProjectionOptions{Tenant: tenant}) + if err != nil { + return nil, err + } + h := &HostedStore{physical: &Store{pg: pg, hostedRelations: true}, core: core, tenant: tenant} + core.curationGuard = h.guardCurationIdentity + if _, err = rand.Read(h.cursorKey[:]); err != nil { + return nil, err + } + return h, nil +} +func (h *HostedStore) DB() *sql.DB { return h.physical.DB() } +func (h *HostedStore) Close() error { return h.physical.Close() } +func (h *HostedStore) SetCustomPricing(p map[string]config.CustomModelRate) { + h.physical.SetCustomPricing(p) +} +func (h *HostedStore) InsightGenerationAvailable() bool { + return h.physical.InsightGenerationAvailable() +} +func (h *HostedStore) DetectInsightGenerationAvailability(ctx context.Context) error { + return h.physical.DetectInsightGenerationAvailability(ctx) +} +func (h *HostedStore) revision(ctx context.Context) (hostedRevision, error) { + var r hostedRevision + err := h.physical.pg.QueryRowContext(ctx, `SELECT identity_revision,selection_revision,corpus_revision FROM raw_corpus_state WHERE singleton=1`).Scan(&r.Identity, &r.Selection, &r.Corpus) + if errors.Is(err, sql.ErrNoRows) { + err = nil + } + return r, err +} + +// Read failures are also revision checked: a disappearing physical row during +// convergence must retry resolution, not report an accidental not-found result. +func hostedRead[T any](ctx context.Context, h *HostedStore, read func(hostedRevision) (T, error)) (T, error) { + var zero T + for range 3 { + before, err := h.revision(ctx) + if err != nil { + return zero, err + } + value, readErr := read(before) + after, err := h.revision(ctx) + if err != nil { + return zero, err + } + if before == after { + return value, readErr + } + } + return zero, ErrHostedIdentityChanged +} +func (h *HostedStore) SetCursorSecret(secret []byte) { + h.cursorMu.Lock() + h.cursorKey = sha256.Sum256(secret) + h.cursorMu.Unlock() + h.physical.SetCursorSecret(secret) +} +func (h *HostedStore) cipher() (cipher.AEAD, error) { + h.cursorMu.RLock() + key := h.cursorKey + h.cursorMu.RUnlock() + block, err := aes.NewCipher(key[:]) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} +func (h *HostedStore) sealCursor(payload string, r hostedRevision) (string, error) { + if payload == "" { + return "", nil + } + c, err := h.cipher() + if err != nil { + return "", err + } + nonce := make([]byte, c.NonceSize()) + if _, err = rand.Read(nonce); err != nil { + return "", err + } + aad := fmt.Sprintf("hosted-v1:%s:%d:%d", h.tenant, r.Selection, r.Identity) + return base64.RawURLEncoding.EncodeToString(c.Seal(append([]byte{1}, nonce...), nonce, []byte(payload), []byte(aad))), nil +} +func (h *HostedStore) openCursor(token string, r hostedRevision) (string, error) { + if token == "" { + return "", nil + } + if len(token) > 32768 { + return "", ErrHostedCursor + } + b, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return "", ErrHostedCursor + } + c, err := h.cipher() + if err != nil { + return "", err + } + if len(b) < 1+c.NonceSize()+c.Overhead() || b[0] != 1 { + return "", ErrHostedCursor + } + aad := fmt.Sprintf("hosted-v1:%s:%d:%d", h.tenant, r.Selection, r.Identity) + p, err := c.Open(nil, b[1:1+c.NonceSize()], b[1+c.NonceSize():], []byte(aad)) + if err != nil { + return "", ErrHostedCursor + } + return string(p), nil +} + +// Public cursor helpers accept public coordinates only; physical cursor work +// stays inside ListSessions/GetSidebarSessionIndex. +func (h *HostedStore) EncodeCursor(c db.SessionCursor) string { + token, err := hostedRead(context.Background(), h, func(r hostedRevision) (string, error) { + id, err := h.resolve(context.Background(), c.ID) + if err != nil { + return "", err + } + if id.SessionID == "" { + return "", ErrHostedCursor + } + physical := c + physical.ID = id.SessionID + return h.sealCursor(h.physical.EncodeCursor(physical), r) + }) + if err != nil { + return "" + } + return token +} +func (h *HostedStore) DecodeCursor(token string) (db.SessionCursor, error) { + return hostedRead(context.Background(), h, func(r hostedRevision) (db.SessionCursor, error) { + p, err := h.openCursor(token, r) + if err != nil { + return db.SessionCursor{}, err + } + c, err := h.physical.DecodeCursor(p) + if err != nil { + return c, err + } + refs := hostedRefs{} + refs.add(&c.ID) + err = refs.mapIDs(context.Background(), h) + return c, err + }) +} +func (h *HostedStore) EncodeActivityReportToken(payload []byte) (string, error) { + r, err := h.revision(context.Background()) + if err != nil { + return "", err + } + return h.sealCursor(string(payload), r) +} +func (h *HostedStore) DecodeActivityReportToken(token string) ([]byte, error) { + r, err := h.revision(context.Background()) + if err != nil { + return nil, err + } + p, err := h.openCursor(token, r) + return []byte(p), err +} + +func (h *HostedStore) SetVectorSearcher(searcher db.VectorSearcher) { + h.physical.SetVectorSearcher(searcher) +} +func (h *HostedStore) SetSemanticUnavailableReason(reason string) { + h.physical.SetSemanticUnavailableReason(reason) +} diff --git a/internal/postgres/hosted_store_pgtest_test.go b/internal/postgres/hosted_store_pgtest_test.go new file mode 100644 index 0000000000..6a4d6d5a20 --- /dev/null +++ b/internal/postgres/hosted_store_pgtest_test.go @@ -0,0 +1,369 @@ +//go:build pgtest + +package postgres + +import ( + "encoding/base64" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/activity" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/rawderive" + "testing" + "time" +) + +// A physical Store passed through the hosted constructor cannot resolve the +// provider alias and leaks storage identities in its list and transcript. +func TestHostedPublicReadBoundary(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("public transcript"))) + raw, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + store, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + session, err := store.GetSession(t.Context(), "codex:portable") + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "codex:portable", session.ID) + messages, err := store.GetAllMessages(t.Context(), session.ID) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "public transcript", messages[0].Content) + assert.Equal(t, "codex:portable", messages[0].SessionID) + require.Len(t, messages[1].ToolCalls, 1) + assert.Equal(t, "codex:portable", messages[1].ToolCalls[0].SessionID) + hidden, err := store.GetSession(t.Context(), raw.SessionID) + require.NoError(t, err) + assert.Nil(t, hidden) + page, err := store.ListSessions(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + require.Len(t, page.Sessions, 1) + assert.Equal(t, "codex:portable", page.Sessions[0].ID) +} + +func TestHostedPublicConflictCurationAndCursor(t *testing.T) { + f := newProjectionFixture(t) + a, accepted := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("first transcript"))) + h, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, h.Close()) }) + h.SetCursorSecret([]byte("synthetic test cursor secret")) + _, err = h.PinMessage("codex:portable", 0, nil) + require.NoError(t, err) + require.NoError(t, h.RenameSession("codex:portable", new("Saved title"))) + _, err = h.StarSession("codex:portable") + require.NoError(t, err) + b, _ := f.accept(t, "device-b", "capture-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("second transcript"))) + _, err = h.GetSession(t.Context(), "codex:portable") + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + require.Len(t, conflict.Variants, 2) + page, err := h.ListSessions(t.Context(), db.SessionFilter{Limit: 1}) + require.NoError(t, err) + require.Len(t, page.Sessions, 1) + require.NotEmpty(t, page.NextCursor) + assert.Equal(t, "Saved title", *page.Sessions[0].DisplayName) + decoded, err := base64.RawURLEncoding.DecodeString(page.NextCursor) + require.NoError(t, err) + assert.NotContains(t, string(decoded), "raw-row-") + next, err := h.ListSessions(t.Context(), db.SessionFilter{Limit: 1, Cursor: page.NextCursor}) + require.NoError(t, err) + require.Len(t, next.Sessions, 1) + assert.NotEqual(t, page.Sessions[0].ID, next.Sessions[0].ID) + _, err = h.ListSessions(t.Context(), db.SessionFilter{Cursor: h.physical.EncodeCursor(db.SessionCursor{ID: "raw-row-forged"})}) + assert.ErrorIs(t, err, ErrHostedCursor) + search, err := h.SearchContent(t.Context(), db.ContentSearchFilter{Pattern: "transcript", ExcludeSessionIDs: []string{"codex:portable"}, IncludeOneShot: true}) + require.NoError(t, err) + assert.Empty(t, search.Matches) + changed, _ := f.accept(t, "device-a", "capture-next", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, changed), changed, projectionOutcome("second transcript"))) + _, err = h.ListSessions(t.Context(), db.SessionFilter{Cursor: page.NextCursor}) + assert.ErrorIs(t, err, ErrHostedCursor) + pins, err := h.ListPinnedMessages(t.Context(), "codex:portable", "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.True(t, pins[0].Unresolved) + require.NotEmpty(t, pins[0].MessageKey) + allPins, e := h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, e) + require.Len(t, allPins, 1) + assert.True(t, allPins[0].Unresolved) + assert.Equal(t, pins[0].MessageKey, allPins[0].MessageKey) + require.NotNil(t, allPins[0].SessionProject) + assert.Equal(t, "synthetic", *allPins[0].SessionProject) + require.NoError(t, h.RemovePinReference(t.Context(), "codex:portable", pins[0].MessageKey)) + pins, err = h.ListPinnedMessages(t.Context(), "codex:portable", "") + require.NoError(t, err) + assert.Empty(t, pins) +} + +func TestHostedLegacyCollisionRevisions(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("raw transcript"))) + h, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, h.Close()) }) + before, err := h.revision(t.Context()) + require.NoError(t, err) + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent,message_count,user_message_count) VALUES('codex:portable','legacy','archive','codex',2,1)`) + require.NoError(t, err) + after, err := h.revision(t.Context()) + require.NoError(t, err) + assert.Greater(t, after.Identity, before.Identity) + _, err = h.GetSession(t.Context(), "codex:portable") + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + require.Len(t, conflict.Variants, 2) + page, err := h.ListSessions(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + require.Len(t, page.Sessions, 2) + for _, s := range page.Sessions { + assert.NotEqual(t, "codex:portable", s.ID) + full, e := h.GetSessionFull(t.Context(), s.ID) + require.NoError(t, e) + require.NotNil(t, full) + assert.Equal(t, s.ID, full.ID) + } + _, err = f.runtime.ExecContext(t.Context(), `UPDATE sessions SET display_name='archive title' WHERE id='codex:portable'`) + require.NoError(t, err) + content, err := h.revision(t.Context()) + require.NoError(t, err) + assert.Equal(t, after.Identity, content.Identity) + assert.Greater(t, content.Corpus, after.Corpus) +} + +func TestHostedSharedChildTraversesBothParentVariants(t *testing.T) { + f := newProjectionFixture(t) + outcome := func(parentText string) rawderive.ParsedManifest { + p := projectionOutcome(parentText) + child := projectionOutcome("shared child").Outcome.Results[0] + child.Result.Session.ID = "codex:child" + child.Result.Session.SourceSessionID = "child" + child.Result.Session.ParentSessionID = "codex:portable" + child.Result.Session.RelationshipType = "subagent" + child.Result.Session.EndedAt = child.Result.Session.StartedAt.Add(7 * time.Second) + p.Outcome.Results[0].Result.Messages[1].ToolCalls[0].SubagentSessionID = "codex:child" + p.Outcome.Results[0].Result.Messages[1].HasToolUse = true + p.Outcome.Results = append(p.Outcome.Results, child) + return p + } + a, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, outcome("parent one"))) + b, acceptedB := f.accept(t, "device-b", "capture-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, outcome("parent two"))) + h, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, h.Close()) }) + _, err = h.GetSession(t.Context(), "codex:portable") + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + child, err := h.GetSession(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, child) + assert.Nil(t, child.ParentSessionID) + assert.Equal(t, conflict.Variants, child.ParentSessionIDs) + sidebar, err := h.GetSidebarSessionIndex(t.Context(), db.SessionFilter{Limit: 1}) + require.NoError(t, err) + assert.Equal(t, 2, sidebar.Total) + require.Len(t, sidebar.Sessions, 2) + require.NotEmpty(t, sidebar.NextCursor) + second, err := h.GetSidebarSessionIndex(t.Context(), db.SessionFilter{Limit: 1, Cursor: sidebar.NextCursor}) + require.NoError(t, err) + require.Len(t, second.Sessions, 2) + for _, parent := range conflict.Variants { + children, e := h.GetChildSessions(t.Context(), parent) + require.NoError(t, e) + require.Len(t, children, 1) + assert.Equal(t, "codex:child", children[0].ID) + require.NotNil(t, children[0].ParentSessionID) + assert.Equal(t, parent, *children[0].ParentSessionID) + messages, e := h.GetAllMessages(t.Context(), parent) + require.NoError(t, e) + require.Len(t, messages, 2) + require.Len(t, messages[1].ToolCalls, 1) + assert.Equal(t, "codex:child", messages[1].ToolCalls[0].SubagentSessionID) + timing, e := h.GetSessionTiming(t.Context(), parent) + require.NoError(t, e) + require.NotNil(t, timing) + require.NotNil(t, timing.SlowestCall) + require.NotNil(t, timing.SlowestCall.DurationMs) + assert.Equal(t, int64(7000), *timing.SlowestCall.DurationMs) + } + removed, _ := f.accept(t, "device-b", "capture-without-parent", acceptedB.Receipt) + withoutParent := outcome("parent two") + withoutParent.Outcome.Results = withoutParent.Outcome.Results[1:] + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, removed), removed, withoutParent)) + child, err = h.GetSession(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, child.ParentSessionID) + assert.Equal(t, "codex:portable", *child.ParentSessionID) + assert.Empty(t, child.ParentSessionIDs) + sidebar, err = h.GetSidebarSessionIndex(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + assert.Equal(t, 1, sidebar.Total) + require.Len(t, sidebar.Sessions, 2) + +} + +func TestHostedReadRetriesTheWholeIdentitySnapshot(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("before"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + reads := 0 + value, err := hostedRead(t.Context(), h, func(_ hostedRevision) (string, error) { + reads++ + target, e := h.resolve(t.Context(), "codex:portable") + if e != nil { + return "", e + } + messages, e := h.physical.GetAllMessages(t.Context(), target.SessionID) + if e != nil { + return "", e + } + if reads == 1 { + next, _ := f.accept(t, "device-a", "capture-b", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("after"))) + } + return messages[0].Content, nil + }) + require.NoError(t, err) + assert.Equal(t, "after", value) + assert.Equal(t, 2, reads) + _, err = hostedRead(t.Context(), h, func(_ hostedRevision) (string, error) { + _, e := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('revision-probe','synthetic','archive','codex') ON CONFLICT(id) DO UPDATE SET display_name='still changing'`) + return "discard", e + }) + assert.ErrorIs(t, err, ErrHostedIdentityChanged) +} + +func TestHostedUsageAndActivityReferences(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + p := projectionOutcome("usage transcript") + r := &p.Outcome.Results[0].Result + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + r.Session.EndedAt = start.Add(time.Minute) + r.Messages[0].Timestamp = start + r.Messages[1].Timestamp = start.Add(time.Minute) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, p)) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + rows, err := h.GetSessionUsageRows(t.Context(), []string{"codex:portable"}) + require.NoError(t, err) + require.NotNil(t, rows) + require.Len(t, rows.Rows, 1) + assert.Equal(t, "codex:portable", rows.Rows[0].SessionID) + assert.Equal(t, "codex:portable", rows.Rows[0].SourceSessionID) + assert.Equal(t, 11, rows.Rows[0].InputTokens) + assert.Equal(t, 7, rows.Rows[0].OutputTokens) + assert.Equal(t, map[string]int{"codex:portable": 7}, rows.RawOutputTokensBySession) + usage, err := h.GetSessionUsage(t.Context(), "codex:portable", true) + require.NoError(t, err) + require.NotNil(t, usage) + assert.Equal(t, "codex:portable", usage.SessionID) + require.Len(t, usage.Breakdown, 1) + assert.Equal(t, 11, usage.Breakdown[0].InputTokens) + assert.Equal(t, 7, usage.Breakdown[0].OutputTokens) + query, err := activity.ResolveQuery(activity.QueryInput{Date: "2026-01-01", Preset: "day", Timezone: "UTC"}, start.Add(48*time.Hour)) + require.NoError(t, err) + artifacts, err := h.BuildActivityReportArtifacts(t.Context(), db.AnalyticsFilter{}, query, nil) + require.NoError(t, err) + require.Len(t, artifacts.Sessions, 1) + assert.Equal(t, "codex:portable", artifacts.Sessions[0].SessionID) + assert.Contains(t, artifacts.Membership, "codex:portable") + for _, interval := range artifacts.Report.Intervals { + assert.Equal(t, "codex:portable", interval.SessionID) + } +} + +func TestHostedCurationGuardChecksCollisionInsideTransaction(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("raw transcript"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('codex:portable','legacy','archive','codex')`) + require.NoError(t, err) + // This bypasses the adapter's optimistic preflight deliberately: the core + // write's configured public guard must still reject the colliding base. + err = h.core.SetCuration(t.Context(), "codex:portable", "starred", true) + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + require.Len(t, conflict.Variants, 2) + stars, err := h.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Empty(t, stars) +} + +func TestHostedBulkCurationResolvesEntireBatchBeforeWriting(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("visible"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + var gone *db.SessionIdentityError + err = h.BulkStarSessions([]string{"codex:portable", "missing"}) + require.ErrorAs(t, err, &gone) + stars, err := h.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Empty(t, stars) + count, err := h.SoftDeleteSessions([]string{"codex:portable", "missing"}) + require.ErrorAs(t, err, &gone) + assert.Zero(t, count) + session, err := h.GetSession(t.Context(), "codex:portable") + require.NoError(t, err) + require.NotNil(t, session) +} + +func TestHostedConstructorRejectsMissingRevisionProtection(t *testing.T) { + f := newHostedFixture(t, "tenant-projection") + _, err := f.admin.ExecContext(t.Context(), `DROP TRIGGER hosted_legacy_revision ON sessions`) + require.NoError(t, err) + _, err = NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.ErrorContains(t, err, "owner must reprovision") +} + +func TestHostedConstructorRejectsWeakenedRevisionProtection(t *testing.T) { + for _, alteration := range []string{ + `DROP TRIGGER hosted_legacy_revision ON sessions; CREATE TRIGGER hosted_legacy_revision AFTER INSERT OR UPDATE OR DELETE ON sessions FOR EACH ROW WHEN (false) EXECUTE FUNCTION hosted_legacy_revision()`, + `CREATE FUNCTION noop_revision() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NULL; END $$; DROP TRIGGER hosted_legacy_revision ON sessions; CREATE TRIGGER hosted_legacy_revision AFTER INSERT OR UPDATE OR DELETE ON sessions FOR EACH ROW EXECUTE FUNCTION noop_revision()`, + `CREATE OR REPLACE FUNCTION hosted_legacy_revision() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NULL; END $$`, + } { + t.Run(alteration[:min(35, len(alteration))], func(t *testing.T) { + f := newHostedFixture(t, "tenant-projection") + _, err := f.admin.ExecContext(t.Context(), alteration) + require.NoError(t, err) + _, err = NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.ErrorContains(t, err, "owner must reprovision") + }) + } +} + +func TestHostedUnresolvedPinRetainsCreationTime(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "pin-time-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("original pinned message"))) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = h.PinMessage("codex:portable", 0, new("retained note")) + require.NoError(t, err) + before, err := h.ListPinnedMessages(t.Context(), "codex:portable", "") + require.NoError(t, err) + require.Len(t, before, 1) + next, _ := f.accept(t, "device-a", "pin-time-b", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("replacement at same ordinal"))) + after, err := h.ListPinnedMessages(t.Context(), "", "") + require.NoError(t, err) + require.Len(t, after, 1) + assert.True(t, after[0].Unresolved) + assert.Equal(t, before[0].CreatedAt, after[0].CreatedAt) + assert.NotEmpty(t, after[0].CreatedAt) +} diff --git a/internal/postgres/hosted_tenant.go b/internal/postgres/hosted_tenant.go new file mode 100644 index 0000000000..3e99fd8e75 --- /dev/null +++ b/internal/postgres/hosted_tenant.go @@ -0,0 +1,319 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "strings" + + archivedb "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/rawsync" +) + +func validateHostedBinding(schema, tenant string) error { + if len(schema) > 63 { + return fmt.Errorf("hosted schema identifier exceeds PostgreSQL limit") + } + if _, err := quoteIdentifier(schema); err != nil { + return err + } + identity, err := rawsync.NewAuthIdentity(tenant, "hosted-validation") + if err != nil || identity.TenantID != tenant { + return fmt.Errorf("hosted tenant must be nonblank and canonical") + } + return nil +} + +// EnsureHostedTenant explicitly adopts a schema for exactly one tenant using a +// separate owner credential. Ordinary Open/EnsureSchema never opt into this. +// Provisioning must run before granting runtime access. Existing raw rows must +// already belong to the configured tenant; legacy derived rows are retained. +func EnsureHostedTenant(ctx context.Context, database *sql.DB, schema, tenant string) error { + if database == nil { + return fmt.Errorf("hosted provisioning requires a database") + } + if err := validateHostedBinding(schema, tenant); err != nil { + return err + } + + tx, err := database.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + // Transaction-scoped locking uses one pool connection, including concurrent + // provisioners and pools bounded to a single connection. + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "agentsview-hosted:"+schema); err != nil { + return err + } + quoted, _ := quoteIdentifier(schema) + if _, err = tx.ExecContext(ctx, `SELECT set_config('search_path',$1,true),set_config('agentsview.tenant_id',$2,true),set_config('standard_conforming_strings','on',true)`, quoted, tenant); err != nil { + return err + } + var exists bool + if err = tx.QueryRowContext(ctx, `SELECT to_regclass(format('%I.hosted_tenant_binding',$1::text)) IS NOT NULL`, schema).Scan(&exists); err != nil { + return err + } + if exists { + if err = checkHostedBinding(ctx, tx, schema, tenant); err != nil { + return err + } + if err = installRawProjectionUpgrade(ctx, tx, schema, tenant); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, hostedLegacyRevisionDDL); err != nil { + return err + } + if err = checkHostedCatalog(ctx, tx, schema, tenant); err != nil { + return err + } + return tx.Commit() + } + if err = preflightHostedAdoption(ctx, tx, schema, tenant); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `CREATE SCHEMA IF NOT EXISTS `+quoted); err != nil { + return err + } + for _, ddl := range []string{coreDDL, rawIngestDDL, rawIngestAppendOnlyDDL, postgresUsageJSONHelperDDL, rawProjectionDDL, rawLinksDDL} { + if _, err = tx.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("creating hosted application schema: %w", err) + } + } + if err = ensureRawProjectionJobColumns(ctx, tx); err != nil { + return err + } + migrations := schemaColumnMigrations() + columns, err := loadExistingColumns(ctx, tx, migrations) + if err != nil { + return err + } + if _, err = ensureColumns(ctx, tx, columns, migrations); err != nil { + return err + } + var dataVersion int + if err = tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(data_version),0) FROM sessions`).Scan(&dataVersion); err != nil { + return err + } + if dataVersion > archivedb.CurrentDataVersion() { + return &archivedb.DataVersionTooNewError{DatabaseVersion: dataVersion, BinaryVersion: archivedb.CurrentDataVersion()} + } + if err = createPartialIndexesPG(ctx, tx); err != nil { + return err + } + + if _, err = tx.ExecContext(ctx, vectorBaseDDL); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `CREATE TABLE hosted_tenant_binding(singleton SMALLINT PRIMARY KEY CHECK(singleton=1),tenant_id TEXT NOT NULL)`); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO hosted_tenant_binding(singleton,tenant_id) VALUES (1,$1)`, tenant); err != nil { + return err + } + if err = checkHostedInventory(ctx, tx, schema); err != nil { + return err + } + if err = InstallHostedTables(ctx, tx, schema, tenant, hostedTables); err != nil { + return fmt.Errorf("installing hosted tenant constraints: %w", err) + } + if _, err = tx.ExecContext(ctx, rawProjectionIndexesDDL); err != nil { + return err + } + // Statement trigger also fences DELETE/TRUNCATE, which a literal CHECK cannot. + if _, err = tx.ExecContext(ctx, `CREATE FUNCTION hosted_reject_binding_mutation() RETURNS trigger LANGUAGE plpgsql AS $binding$ BEGIN RAISE EXCEPTION 'hosted tenant binding is immutable' USING ERRCODE='55000'; END; $binding$; + CREATE TRIGGER hosted_binding_immutable BEFORE UPDATE OR DELETE OR TRUNCATE ON hosted_tenant_binding FOR EACH STATEMENT EXECUTE FUNCTION hosted_reject_binding_mutation()`); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, hostedLegacyRevisionDDL); err != nil { + return err + } + if err = checkHostedCatalog(ctx, tx, schema, tenant); err != nil { + return err + } + return tx.Commit() +} + +type hostedQuerier interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func checkHostedBinding(ctx context.Context, q hostedQuerier, schema, tenant string) error { + qs, _ := quoteIdentifier(schema) + var got string + if err := q.QueryRowContext(ctx, `SELECT tenant_id FROM `+qs+`.hosted_tenant_binding WHERE singleton=1`).Scan(&got); err != nil { + return fmt.Errorf("reading hosted tenant binding: %w", err) + } + if got != tenant { + return fmt.Errorf("configured tenant does not match permanent schema binding") + } + return nil +} +func checkHostedInventory(ctx context.Context, q hostedQuerier, schema string) error { + expected := make(map[string]bool, len(hostedTables)) + for _, t := range hostedTables { + expected[t.Name] = true + } + rows, err := q.QueryContext(ctx, `SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind IN ('r','p','v','m','f')`, schema) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + return err + } + if !expected[name] && strings.HasPrefix(name, "vector_") { + return fmt.Errorf("hosted vector schema is unsupported") + } + if !expected[name] { + return fmt.Errorf("hosted schema contains uninventoried relation %s", name) + } + delete(expected, name) + } + if err = rows.Err(); err != nil { + return err + } + if len(expected) > 0 { + return fmt.Errorf("hosted schema is missing required tables") + } + return nil +} + +func checkHostedCatalog(ctx context.Context, q hostedQuerier, schema, tenant string) error { + if err := checkHostedInventory(ctx, q, schema); err != nil { + return err + } + for _, table := range hostedTables { + var good bool + err := q.QueryRowContext(ctx, `SELECT c.relkind='r' AND c.relrowsecurity AND c.relforcerowsecurity + AND EXISTS(SELECT 1 FROM pg_attribute a WHERE a.attrelid=c.oid AND a.attname='tenant_id' AND a.attnotnull AND a.atttypid='text'::regtype) + AND EXISTS(SELECT 1 FROM pg_constraint k WHERE k.conrelid=c.oid AND k.contype='p' AND array_to_string(ARRAY(SELECT a.attname FROM unnest(k.conkey) WITH ORDINALITY x(n,i) JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum=x.n ORDER BY x.i),',')=$3) + AND EXISTS(SELECT 1 FROM pg_constraint k WHERE k.conrelid=c.oid AND k.contype='c' AND k.convalidated AND k.conname='hosted_tenant_check' AND pg_get_expr(k.conbin,k.conrelid)=format('(tenant_id = %L::text)',$4::text)) + AND (SELECT count(*) FROM pg_policy p WHERE p.polrelid=c.oid)=1 + AND EXISTS(SELECT 1 FROM pg_policy p WHERE p.polrelid=c.oid AND p.polname='hosted_tenant_policy' AND p.polcmd='*' AND p.polroles=ARRAY[0::oid] AND pg_get_expr(p.polqual,p.polrelid)=$5 AND pg_get_expr(p.polwithcheck,p.polrelid)=$5) + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relname=$2`, schema, table.Name, "tenant_id,"+strings.Join(table.Key, ","), tenant, `(tenant_id = current_setting('agentsview.tenant_id'::text, true))`).Scan(&good) + if err != nil { + return err + } + if !good { + return fmt.Errorf("hosted protections missing or altered on %s", table.Name) + } + var count int + if err = q.QueryRowContext(ctx, `SELECT count(*) FROM pg_constraint WHERE conrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND contype='f'`, schema, table.Name).Scan(&count); err != nil { + return err + } + if count != len(table.ForeignKeys) { + return fmt.Errorf("hosted foreign key inventory mismatch on %s", table.Name) + } + for _, fk := range table.ForeignKeys { + del, _ := hostedAction(fk.Delete) + upd, _ := hostedAction(fk.Update) + err = q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_constraint c WHERE c.conrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND c.confrelid=to_regclass(format('%I.%I',$1::text,$3::text)) AND c.contype='f' AND c.convalidated AND c.confdeltype::text=$4 AND c.confupdtype::text=$5 AND array_to_string(ARRAY(SELECT a.attname FROM unnest(c.conkey) WITH ORDINALITY k(n,i) JOIN pg_attribute a ON a.attrelid=c.conrelid AND a.attnum=k.n ORDER BY k.i),',')=$6 AND array_to_string(ARRAY(SELECT a.attname FROM unnest(c.confkey) WITH ORDINALITY k(n,i) JOIN pg_attribute a ON a.attrelid=c.confrelid AND a.attnum=k.n ORDER BY k.i),',')=$7)`, schema, table.Name, fk.Table, hostedActionCode(del), hostedActionCode(upd), "tenant_id,"+strings.Join(fk.Columns, ","), "tenant_id,"+strings.Join(fk.References, ",")).Scan(&good) + if err != nil { + return err + } + if !good { + return fmt.Errorf("hosted foreign key protection missing on %s", table.Name) + } + } + } + var immutable bool + if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid=to_regclass(format('%I.hosted_tenant_binding',$1::text)) AND tgname='hosted_binding_immutable' AND tgenabled='O')`, schema).Scan(&immutable); err != nil { + return err + } + if !immutable { + return fmt.Errorf("hosted binding immutability trigger missing") + } + return nil +} + +// CheckHostedTenant is a read-only startup gate for a permanently bound pool. +// Runtime ownership, role switching, schema creation and sibling access are +// rejected; PostgreSQL RLS support is mandatory in hosted mode. +func CheckHostedTenant(ctx context.Context, database *sql.DB, schema, tenant string) error { + if database == nil { + return fmt.Errorf("hosted check requires a database") + } + if err := validateHostedBinding(schema, tenant); err != nil { + return err + } + conn, err := database.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var currentSchema, currentTenant, searchPath string + if err = conn.QueryRowContext(ctx, `SELECT COALESCE(current_schema(),''),COALESCE(current_setting('agentsview.tenant_id',true),''),current_setting('search_path')`).Scan(¤tSchema, ¤tTenant, &searchPath); err != nil { + return err + } + quoted, _ := quoteIdentifier(schema) + if currentSchema != schema || currentTenant != tenant || searchPath != quoted+", pg_temp" { + return fmt.Errorf("hosted connection schema or tenant context mismatch") + } + var unsafe bool + if err = conn.QueryRowContext(ctx, `SELECT r.rolsuper OR r.rolbypassrls OR r.rolcreaterole OR r.rolcreatedb OR r.rolreplication OR current_user<>session_user + OR EXISTS(SELECT 1 FROM pg_auth_members WHERE member=r.oid) + OR has_database_privilege(current_database(),'CREATE') + OR EXISTS(SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE p.prosecdef AND n.nspname !~ '^pg_' AND n.nspname<>'information_schema' AND has_function_privilege(p.oid,'EXECUTE')) + OR EXISTS(SELECT 1 FROM pg_namespace n WHERE n.nspname !~ '^pg_' AND n.nspname<>'information_schema' AND (n.nspowner=r.oid OR has_schema_privilege(n.oid,'CREATE') OR (n.nspname<>$1 AND n.nspname<>'public' AND has_schema_privilege(n.oid,'USAGE')))) + OR EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind IN ('r','p') AND (has_table_privilege(c.oid,'TRUNCATE,TRIGGER,REFERENCES') OR has_any_column_privilege(c.oid,'REFERENCES'))) + OR EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname !~ '^pg_' AND n.nspname<>'information_schema' AND c.relkind IN ('r','p','v','m','f','S') AND (c.relowner=r.oid OR (n.nspname<>$1 AND CASE WHEN c.relkind='S' THEN has_sequence_privilege(c.oid,'USAGE,SELECT,UPDATE') ELSE (has_table_privilege(c.oid,'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') OR has_any_column_privilege(c.oid,'SELECT,INSERT,UPDATE,REFERENCES')) END))) + FROM pg_roles r WHERE r.rolname=current_user`, schema).Scan(&unsafe); err != nil { + return err + } + if unsafe { + return fmt.Errorf("hosted runtime requires a restricted non-owner role without sibling access or DDL privileges") + } + if err = checkHostedBinding(ctx, conn, schema, tenant); err != nil { + return err + } + return checkHostedCatalog(ctx, conn, schema, tenant) +} + +func preflightHostedAdoption(ctx context.Context, q hostedQuerier, schema, tenant string) error { + known := make(map[string]bool) + for _, table := range hostedTables { + known[table.Name] = true + } + rows, err := q.QueryContext(ctx, `SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind IN ('r','p','v','m','f')`, schema) + if err != nil { + return err + } + var names []string + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + rows.Close() + return err + } + names = append(names, name) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + qs, _ := quoteIdentifier(schema) + for _, name := range names { + if strings.HasPrefix(name, "vector_") && !known[name] { + return fmt.Errorf("hosted vector schema is unsupported") + } + if !known[name] { + return fmt.Errorf("hosted schema contains uninventoried relation %s", name) + } + if strings.HasPrefix(name, "raw_") { + qt, _ := quoteIdentifier(name) + var mismatch bool + if err = q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+qs+`.`+qt+` WHERE tenant_id IS DISTINCT FROM $1)`, tenant).Scan(&mismatch); err != nil { + return err + } + if mismatch { + return fmt.Errorf("existing raw rows do not match configured hosted tenant") + } + } + } + return nil +} diff --git a/internal/postgres/hosted_tenant_pgtest_test.go b/internal/postgres/hosted_tenant_pgtest_test.go new file mode 100644 index 0000000000..7d9f57bb6e --- /dev/null +++ b/internal/postgres/hosted_tenant_pgtest_test.go @@ -0,0 +1,491 @@ +//go:build pgtest + +package postgres + +import ( + "context" + cryptorand "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "math/rand/v2" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/rawsync" +) + +type hostedFixture struct { + admin, runtime *sql.DB + schema, role, tenant, dsn string +} + +func newHostedFixture(t *testing.T, tenant string) hostedFixture { + t.Helper() + schema := fmt.Sprintf("hosted_test_%x", rand.Uint64()) + role := schema + "_runtime" + admin, err := Open(testPGURL(t), schema, false) + require.NoError(t, err) + t.Cleanup(func() { + _, err := admin.ExecContext(context.Background(), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`) + assert.NoError(t, err) + _, err = admin.ExecContext(context.Background(), `DROP ROLE IF EXISTS "`+role+`"`) + assert.NoError(t, err) + assert.NoError(t, admin.Close()) + }) + require.NoError(t, EnsureHostedTenant(t.Context(), admin, schema, tenant)) + _, err = admin.ExecContext(t.Context(), `CREATE ROLE "`+role+`" LOGIN NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOINHERIT`) + require.NoError(t, err) + _, err = admin.ExecContext(t.Context(), `GRANT USAGE ON SCHEMA "`+schema+`" TO "`+role+`"; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA "`+schema+`" TO "`+role+`"; + GRANT USAGE ON ALL SEQUENCES IN SCHEMA "`+schema+`" TO "`+role+`"`) + require.NoError(t, err) + password := setHostedFixturePassword(t, admin, role) + dsn, err := appendConnParams(testPGURL(t), map[string]string{"password": password, "user": role, "search_path": "public", "agentsview.tenant_id": "wrong"}) + require.NoError(t, err) + runtime, err := OpenHosted(dsn, schema, tenant, false) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, runtime.Close()) }) + return hostedFixture{admin, runtime, schema, role, tenant, dsn} +} + +func TestHostedTenantIsolation(t *testing.T) { + a := newHostedFixture(t, "tenant-a") + b := newHostedFixture(t, "tenant-b") + for _, f := range []hostedFixture{a, b} { + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES ('same-id',$1,'device','claude')`, f.tenant) + require.NoError(t, err) + var got string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT project FROM sessions WHERE id='same-id'`).Scan(&got)) + assert.Equal(t, f.tenant, got) + require.NoError(t, EnsureHostedTenant(t.Context(), f.admin, f.schema, f.tenant)) + assert.Error(t, EnsureHostedTenant(t.Context(), f.admin, f.schema, "different")) + } + _, err := a.runtime.ExecContext(t.Context(), `SELECT * FROM "`+b.schema+`".sessions`) + assert.Error(t, err) + _, err = a.runtime.ExecContext(t.Context(), `INSERT INTO sessions(tenant_id,id,project,machine,agent) VALUES ('tenant-b','bad','p','m','claude')`) + assert.Error(t, err) + // A caller changing the setting cannot escape either the RLS or literal CHECK. + conn, err := a.runtime.Conn(t.Context()) + require.NoError(t, err) + for _, value := range []string{"", "tenant-b"} { + _, err = conn.ExecContext(t.Context(), `SELECT set_config('agentsview.tenant_id',$1,false)`, value) + require.NoError(t, err) + var count int + require.NoError(t, conn.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Zero(t, count) + _, err = conn.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES ('bad','p','m','claude')`) + assert.Error(t, err) + } + require.NoError(t, conn.Close()) + require.NoError(t, CheckHostedTenant(t.Context(), a.runtime, a.schema, a.tenant)) + assert.Error(t, CheckHostedTenant(t.Context(), a.admin, a.schema, a.tenant)) + assert.Error(t, CheckHostedTenant(t.Context(), a.runtime, b.schema, a.tenant)) + // Binding metadata must not be mutable even with table DML grants. + _, err = a.runtime.ExecContext(t.Context(), `UPDATE hosted_tenant_binding SET tenant_id='tenant-b'`) + assert.Error(t, err) + _, err = a.runtime.ExecContext(t.Context(), `DELETE FROM hosted_tenant_binding`) + assert.Error(t, err) +} + +func TestHostedTenantPoolAndCatalog(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + assert.Equal(t, 5, f.runtime.Stats().MaxOpenConnections) + for round := 0; round < 2; round++ { + var connections []*sql.Conn + for range 5 { + c, err := f.runtime.Conn(t.Context()) + require.NoError(t, err) + connections = append(connections, c) + var schema, tenant string + require.NoError(t, c.QueryRowContext(t.Context(), `SELECT current_schema(),current_setting('agentsview.tenant_id')`).Scan(&schema, &tenant)) + assert.Equal(t, f.schema, schema) + assert.Equal(t, f.tenant, tenant) + } + for _, c := range connections { + require.NoError(t, c.Close()) + } + f.runtime.SetConnMaxLifetime(time.Nanosecond) + } + var unsafe int + require.NoError(t, f.admin.QueryRowContext(t.Context(), `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind='r' AND (NOT c.relrowsecurity OR NOT c.relforcerowsecurity OR NOT EXISTS (SELECT 1 FROM pg_attribute a WHERE a.attrelid=c.oid AND a.attname='tenant_id' AND a.attnotnull) OR NOT EXISTS (SELECT 1 FROM pg_constraint k JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum=k.conkey[1] WHERE k.conrelid=c.oid AND k.contype='p' AND a.attname='tenant_id'))`, f.schema).Scan(&unsafe)) + assert.Zero(t, unsafe) + require.NoError(t, f.admin.QueryRowContext(t.Context(), `SELECT count(*) FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND k.contype='f' AND ((SELECT attname FROM pg_attribute WHERE attrelid=k.conrelid AND attnum=k.conkey[1]) <> 'tenant_id' OR (SELECT attname FROM pg_attribute WHERE attrelid=k.confrelid AND attnum=k.confkey[1]) <> 'tenant_id')`, f.schema).Scan(&unsafe)) + assert.Zero(t, unsafe) + _, err := f.admin.ExecContext(t.Context(), `ALTER TABLE sessions NO FORCE ROW LEVEL SECURITY`) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) +} + +func TestHostedTenantRejectsForeignRawRowsAndUnknownTables(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + _, err := pg.ExecContext(t.Context(), `INSERT INTO raw_objects(tenant_id,sha256,size_bytes) VALUES ('tenant-b',repeat('a',64),1)`) + require.NoError(t, err) + assert.Error(t, EnsureHostedTenant(t.Context(), pg, schema, "tenant-a")) + var tenant string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT tenant_id FROM raw_objects`).Scan(&tenant)) + assert.Equal(t, "tenant-b", tenant) + f := newHostedFixture(t, "tenant-a") + _, err = f.admin.ExecContext(t.Context(), `CREATE TABLE untracked_data(id integer)`) + require.NoError(t, err) + assert.Error(t, EnsureHostedTenant(t.Context(), f.admin, f.schema, f.tenant)) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) +} + +func TestHostedTenantBoundCustodyRejectsMismatch(t *testing.T) { + pg := newHostedLegacyFixture(t) + s, err := NewTenantRawIngestStore(pg, "tenant-a") + require.NoError(t, err) + identity := rawIngestIdentity(t, "tenant-b") + assert.ErrorIs(t, s.RecordVerifiedObjects(t.Context(), identity, nil), rawsync.ErrUnauthorized) + _, err = NewTenantRawIngestStore(pg, "") + assert.Error(t, err) +} + +func TestHostedTenantAuthAndJobFences(t *testing.T) { + // Deliberately use an unrestricted legacy pool: explicit library fences must + // work independently of the hosted role's redundant RLS enforcement. + pg := newHostedLegacyFixture(t) + legacy, err := NewRawIngestStore(pg) + require.NoError(t, err) + auth, err := NewRawDeviceAuthStore(pg) + require.NoError(t, err) + service, err := rawsync.NewDeviceAuthService(auth, time.Hour) + require.NoError(t, err) + first, err := service.EnrollDevice(t.Context(), "tenant-a", "first") + require.NoError(t, err) + second, err := service.EnrollDevice(t.Context(), "tenant-b", "second") + require.NoError(t, err) + boundAuth, err := NewTenantRawDeviceAuthStore(pg, "tenant-a") + require.NoError(t, err) + boundService, err := rawsync.NewDeviceAuthService(boundAuth, time.Hour) + require.NoError(t, err) + _, err = boundService.AuthenticateCredential(t.Context(), first.Identity.DeviceID, first.Credential) + require.NoError(t, err) + _, err = boundService.AuthenticateCredential(t.Context(), second.Identity.DeviceID, second.Credential) + assert.ErrorIs(t, err, rawsync.ErrUnauthorized) + _, err = boundService.IssueToken(t.Context(), second.Identity.DeviceID, second.Credential, rawsync.ScopeAll) + assert.ErrorIs(t, err, rawsync.ErrUnauthorized) + foreignToken, err := service.IssueToken(t.Context(), second.Identity.DeviceID, second.Credential, rawsync.ScopeAll) + require.NoError(t, err) + _, err = boundService.AuthenticateToken(t.Context(), foreignToken.Token, rawsync.ScopeStatus) + assert.ErrorIs(t, err, rawsync.ErrUnauthorized) + _, err = boundService.EnrollDevice(t.Context(), "tenant-b", "foreign") + assert.ErrorIs(t, err, rawsync.ErrUnauthorized) + for _, identity := range []rawsync.AuthIdentity{first.Identity, second.Identity} { + object := rawIngestObject(t, "a", 1) + require.NoError(t, legacy.RecordVerifiedObject(t.Context(), identity, object)) + manifest := rawIngestManifest(t, identity, "capture-a", "", rawIngestCapturedAt(), object) + _, err = legacy.CommitManifest(t.Context(), manifest, "v1") + require.NoError(t, err) + } + bound, err := NewTenantRawIngestStore(pg, "tenant-a") + require.NoError(t, err) + leases, err := bound.ClaimRawParseJobs(t.Context(), "worker", 2, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + assert.Equal(t, "tenant-a", leases[0].Identity.TenantID) + other, err := NewTenantRawIngestStore(pg, "tenant-b") + require.NoError(t, err) + foreign, err := other.ClaimRawParseJobs(t.Context(), "worker", 2, time.Minute) + require.NoError(t, err) + require.Len(t, foreign, 1) + assert.Error(t, bound.CompleteRawParseJob(t.Context(), foreign[0])) + assert.Error(t, bound.HeartbeatRawParseJob(t.Context(), foreign[0], time.Minute)) + assert.Error(t, bound.RetryRawParseJob(t.Context(), foreign[0], time.Now(), "retry", "retry")) + assert.Error(t, bound.FailRawParseJob(t.Context(), foreign[0], "failed", "failed")) + require.NoError(t, bound.HeartbeatRawParseJob(t.Context(), leases[0], time.Minute)) + require.NoError(t, bound.CompleteRawParseJob(t.Context(), leases[0])) + var state string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE tenant_id='tenant-b'`).Scan(&state)) + assert.Equal(t, "leased", state) + _, err = pg.ExecContext(t.Context(), `UPDATE raw_source_heads SET manifest_id=NULL,receipt=NULL,generation=0 WHERE tenant_id='tenant-b'; UPDATE raw_ingest_jobs SET state='ready',lease_owner='',lease_expires_at=NULL WHERE tenant_id='tenant-b'`) + require.NoError(t, err) + leases, err = bound.ClaimRawParseJobs(t.Context(), "worker", 2, time.Minute) + require.NoError(t, err) + assert.Empty(t, leases) + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE tenant_id='tenant-b'`).Scan(&state)) + assert.Equal(t, "ready", state, "tenant A idle cleanup cannot retire tenant B obsolete jobs") + +} + +func TestHostedTenantRejectsDynamicVectorStorageBeforeMutation(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + _, err := pg.ExecContext(t.Context(), `CREATE TABLE vector_chunks_123(doc_key text PRIMARY KEY); INSERT INTO vector_chunks_123 VALUES ('kept')`) + require.NoError(t, err) + require.ErrorContains(t, EnsureHostedTenant(t.Context(), pg, schema, "tenant-a"), "hosted vector schema is unsupported") + var value string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT doc_key FROM vector_chunks_123`).Scan(&value)) + assert.Equal(t, "kept", value) + var changed bool + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT to_regclass('hosted_tenant_binding') IS NOT NULL OR EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid='sessions'::regclass AND attname='tenant_id')`).Scan(&changed)) + assert.False(t, changed) +} + +func TestHostedTenantRuntimeRoleGate(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + for _, test := range []struct{ name, grant, revoke string }{ + {"bypass", `ALTER ROLE "` + f.role + `" BYPASSRLS`, `ALTER ROLE "` + f.role + `" NOBYPASSRLS`}, + {"superuser", `ALTER ROLE "` + f.role + `" SUPERUSER`, `ALTER ROLE "` + f.role + `" NOSUPERUSER`}, + {"ddl", `GRANT CREATE ON SCHEMA "` + f.schema + `" TO "` + f.role + `"`, `REVOKE CREATE ON SCHEMA "` + f.schema + `" FROM "` + f.role + `"`}, + {"truncate", `GRANT TRUNCATE ON sessions TO "` + f.role + `"`, `REVOKE TRUNCATE ON sessions FROM "` + f.role + `"`}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := f.admin.ExecContext(t.Context(), test.grant) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + _, err = f.admin.ExecContext(t.Context(), test.revoke) + require.NoError(t, err) + require.NoError(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + }) + } + sibling := newHostedFixture(t, "tenant-b") + _, err := f.admin.ExecContext(t.Context(), `GRANT "`+sibling.role+`" TO "`+f.role+`"`) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + _, err = f.admin.ExecContext(t.Context(), `REVOKE "`+sibling.role+`" FROM "`+f.role+`"`) + require.NoError(t, err) + _, err = f.admin.ExecContext(t.Context(), `GRANT USAGE ON SCHEMA "`+sibling.schema+`" TO "`+f.role+`"`) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + _, err = f.admin.ExecContext(t.Context(), `REVOKE USAGE ON SCHEMA "`+sibling.schema+`" FROM "`+f.role+`"`) + require.NoError(t, err) +} + +func TestHostedTenantAdoptsLegacyAndEnforcesRelationships(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + _, err := pg.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('preserved','project','device','claude'); INSERT INTO starred_sessions(session_id) VALUES('preserved')`) + require.NoError(t, err) + require.NoError(t, EnsureHostedTenant(t.Context(), pg, schema, "tenant-a")) + var id, tenant string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT session_id,tenant_id FROM starred_sessions`).Scan(&id, &tenant)) + assert.Equal(t, "preserved", id) + assert.Equal(t, "tenant-a", tenant) + _, err = pg.ExecContext(t.Context(), `INSERT INTO starred_sessions(tenant_id,session_id) VALUES('tenant-b','preserved')`) + assert.Error(t, err) + // Missing device/source relationships are enforced even for an owner. + _, err = pg.ExecContext(t.Context(), `INSERT INTO raw_source_heads(tenant_id,device_id,provider,configured_root_id,source_key,source_key_sha256) VALUES('tenant-a','missing','claude','root','source',repeat('a',64))`) + assert.Error(t, err) + _, err = pg.ExecContext(t.Context(), `DELETE FROM sessions WHERE id='preserved'`) + require.NoError(t, err) + var count int + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT count(*) FROM starred_sessions`).Scan(&count)) + assert.Zero(t, count, "composite FK retains legacy CASCADE") +} + +func newHostedLegacyFixture(t *testing.T) *sql.DB { + t.Helper() + schema := fmt.Sprintf("hosted_legacy_%x", rand.Uint64()) + pg, err := Open(testPGURL(t), schema, false) + require.NoError(t, err) + t.Cleanup(func() { + _, err := pg.ExecContext(context.Background(), `DROP SCHEMA "`+schema+`" CASCADE`) + assert.NoError(t, err) + assert.NoError(t, pg.Close()) + }) + require.NoError(t, EnsureSchema(t.Context(), pg, schema)) + return pg +} + +func TestHostedTenantCustodyRoundTrip(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + auth, err := NewTenantRawDeviceAuthStore(f.runtime, f.tenant) + require.NoError(t, err) + service, err := rawsync.NewDeviceAuthService(auth, time.Hour) + require.NoError(t, err) + enrollment, err := service.EnrollDevice(t.Context(), f.tenant, "device") + require.NoError(t, err) + custody, err := NewTenantRawIngestStore(f.runtime, f.tenant) + require.NoError(t, err) + object := rawIngestObject(t, "a", 1) + require.NoError(t, custody.RecordVerifiedObject(t.Context(), enrollment.Identity, object)) + manifest := rawIngestManifest(t, enrollment.Identity, "capture-a", "", rawIngestCapturedAt(), object) + _, err = custody.CommitManifest(t.Context(), manifest, "v1") + require.NoError(t, err) + leases, err := custody.ClaimRawParseJobs(t.Context(), "worker", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + require.NoError(t, custody.CompleteRawParseJob(t.Context(), leases[0])) + var state string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs`).Scan(&state)) + assert.Equal(t, "complete", state) +} + +func TestHostedTenantReadStore(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('read-id','kept','device','claude')`) + require.NoError(t, err) + store, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + defer store.Close() + session, err := store.GetSession(t.Context(), "read-id") + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "kept", session.Project) + _, err = NewHostedStore(f.dsn, f.schema, "tenant-b", false) + assert.Error(t, err) +} + +func TestHostedTenantRejectsPrivilegedFunctions(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + _, err := f.admin.ExecContext(t.Context(), `CREATE FUNCTION privileged_read() RETURNS integer LANGUAGE sql SECURITY DEFINER AS 'SELECT 1'`) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) +} + +func TestHostedTenantProvisioningUsesOneConnection(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + pg.SetMaxOpenConns(1) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + require.NoError(t, EnsureHostedTenant(ctx, pg, schema, "tenant-a")) +} + +func TestHostedTenantConcurrentProvisioning(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + done := make(chan error, 5) + for range 5 { + go func() { done <- EnsureHostedTenant(ctx, pg, schema, "tenant-a") }() + } + for range 5 { + require.NoError(t, <-done) + } + var count int + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT count(*) FROM hosted_tenant_binding`).Scan(&count)) + assert.Equal(t, 1, count) +} + +func TestHostedTenantFailedAdoptionRollsBack(t *testing.T) { + pg := newHostedLegacyFixture(t) + var schema string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT current_schema()`).Scan(&schema)) + legacy, err := NewRawIngestStore(pg) + require.NoError(t, err) + identity := rawIngestIdentity(t, "tenant-a") + object := rawIngestObject(t, "a", 1) + require.NoError(t, legacy.RecordVerifiedObject(t.Context(), identity, object)) + manifest := rawIngestManifest(t, identity, "capture-a", "", rawIngestCapturedAt(), object) + _, err = legacy.CommitManifest(t.Context(), manifest, "v1") + require.NoError(t, err) + // Legacy library callers may have stored custody before device enrollment. + // The new required relationship cannot invent an authenticated device. + require.ErrorContains(t, EnsureHostedTenant(t.Context(), pg, schema, "tenant-a"), "foreign key") + var changed bool + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT to_regclass('hosted_tenant_binding') IS NOT NULL OR EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid='sessions'::regclass AND attname='tenant_id')`).Scan(&changed)) + assert.False(t, changed) + var count int + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_manifests`).Scan(&count)) + assert.Equal(t, 1, count) +} + +func TestHostedTenantRejectsColumnOnlyPrivileges(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + table := fmt.Sprintf("hosted_column_%x", rand.Uint64()) + _, err := f.admin.ExecContext(t.Context(), `CREATE TABLE public."`+table+`"(value text); INSERT INTO public."`+table+`" VALUES ('sibling-row')`) + require.NoError(t, err) + t.Cleanup(func() { + _, err := f.admin.ExecContext(context.Background(), `DROP TABLE public."`+table+`"`) + assert.NoError(t, err) + }) + for _, privilege := range []string{"SELECT", "INSERT", "UPDATE", "REFERENCES"} { + t.Run(privilege, func(t *testing.T) { + _, err := f.admin.ExecContext(t.Context(), `GRANT `+privilege+`(value) ON public."`+table+`" TO "`+f.role+`"`) + require.NoError(t, err) + if privilege == "SELECT" { + var value string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT value FROM public."`+table+`"`).Scan(&value)) + assert.Equal(t, "sibling-row", value) + } + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + _, err = f.admin.ExecContext(t.Context(), `REVOKE `+privilege+`(value) ON public."`+table+`" FROM "`+f.role+`"`) + require.NoError(t, err) + require.NoError(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + }) + } + _, err = f.admin.ExecContext(t.Context(), `GRANT REFERENCES(id) ON sessions TO "`+f.role+`"`) + require.NoError(t, err) + assert.Error(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) +} + +func TestHostedTenantTemporaryTableCannotShadowSessions(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + f.runtime.SetMaxOpenConns(1) + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('fixed-id','protected','device','claude')`) + require.NoError(t, err) + conn, err := f.runtime.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + var beforePID int + require.NoError(t, conn.QueryRowContext(t.Context(), `SELECT pg_backend_pid()`).Scan(&beforePID)) + _, err = conn.ExecContext(t.Context(), `CREATE TEMPORARY TABLE sessions(id text,project text); INSERT INTO pg_temp.sessions VALUES('fixed-id','shadow')`) + require.NoError(t, err) + var project string + require.NoError(t, conn.QueryRowContext(t.Context(), `SELECT project FROM sessions WHERE id='fixed-id'`).Scan(&project)) + assert.Equal(t, "protected", project) + require.NoError(t, conn.Close()) + require.NoError(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) + var afterPID int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT pg_backend_pid(),project FROM sessions WHERE id='fixed-id'`).Scan(&afterPID, &project)) + assert.Equal(t, beforePID, afterPID, "exercise reuse of the connection holding the temporary table") + assert.Equal(t, "protected", project) +} + +func TestHostedTenantFixtureUsesSCRAMCredential(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + var hasVerifier bool + require.NoError(t, f.admin.QueryRowContext(t.Context(), `SELECT COALESCE(rolpassword LIKE 'SCRAM-SHA-256$%',false) FROM pg_authid WHERE rolname=$1`, f.role).Scan(&hasVerifier)) + assert.True(t, hasVerifier, "runtime fixture must authenticate on SCRAM test databases") +} + +// setHostedFixturePassword supplies a fresh credential for SCRAM test servers. +// Suppress statement/parameter logging only in this provisioning transaction; +// the secret is bound as a parameter and never appears in a SQL literal here. +func setHostedFixturePassword(t *testing.T, admin *sql.DB, role string) string { + t.Helper() + var random [32]byte + _, err := cryptorand.Read(random[:]) + require.NoError(t, err) + password := hex.EncodeToString(random[:]) + tx, err := admin.BeginTx(t.Context(), nil) + require.NoError(t, err) + defer tx.Rollback() + _, err = tx.ExecContext(t.Context(), `SET LOCAL log_statement='none'; + SET LOCAL log_min_duration_statement=-1; SET LOCAL log_min_duration_sample=-1; + SET LOCAL log_min_error_statement='panic'; SET LOCAL log_parameter_max_length=0; + SET LOCAL log_parameter_max_length_on_error=0; SET LOCAL password_encryption='scram-sha-256'`) + require.NoError(t, err) + _, err = tx.ExecContext(t.Context(), `SELECT set_config('hosted_test.role',$1,true),set_config('hosted_test.password',$2,true)`, role, password) + require.True(t, err == nil, "binding synthetic runtime credential failed") + _, err = tx.ExecContext(t.Context(), `DO $credential$ BEGIN + EXECUTE format('ALTER ROLE %I PASSWORD %L',current_setting('hosted_test.role'),current_setting('hosted_test.password')); + END $credential$`) + require.True(t, err == nil, "provisioning synthetic runtime credential failed") + require.NoError(t, tx.Commit()) + return password +} + +func TestHostedTenantRejectsIncompleteSearchPath(t *testing.T) { + f := newHostedFixture(t, "tenant-a") + quoted, err := quoteIdentifier(f.schema) + require.NoError(t, err) + dsn, err := appendConnParams(f.dsn, map[string]string{"search_path": quoted, "agentsview.tenant_id": f.tenant}) + require.NoError(t, err) + unpinned, err := sql.Open("pgx", dsn) + require.NoError(t, err) + defer unpinned.Close() + assert.Error(t, CheckHostedTenant(t.Context(), unpinned, f.schema, f.tenant), "schema-only path leaves temporary relations implicitly first") +} diff --git a/internal/postgres/hosted_tenant_schema.go b/internal/postgres/hosted_tenant_schema.go new file mode 100644 index 0000000000..96ccda88dc --- /dev/null +++ b/internal/postgres/hosted_tenant_schema.go @@ -0,0 +1,260 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// HostedTable describes a tenant-owned table. Keys omit tenant_id; the installer +// always prefixes it. Extend the inventory with projection tables when their +// schema lands. Unknown tables (including vector storage) fail closed. +type HostedTable struct { + Name string + Key []string + ForeignKeys []HostedForeignKey +} + +// HostedForeignKey describes the complete relationship, excluding tenant_id. +// Delete and Update must be one of the PostgreSQL referential actions below. +type HostedForeignKey struct { + Columns []string + Table string + References []string + Delete string + Update string +} + +var hostedTables = append([]HostedTable{ + {Name: "vector_generations", Key: []string{"id"}}, + {Name: "vector_generation_machines", Key: []string{"generation_id", "machine"}}, + {Name: "vector_documents", Key: []string{"doc_key"}}, + {Name: "vector_push_state", Key: []string{"generation_id", "session_id"}}, + {Name: "hosted_tenant_binding", Key: []string{"singleton"}}, + {Name: "sync_metadata", Key: []string{"key"}}, + {Name: "sessions", Key: []string{"id"}}, + {Name: "messages", Key: []string{"session_id", "ordinal"}, ForeignKeys: sessionHostedFK()}, + {Name: "usage_events", Key: []string{"id"}, ForeignKeys: sessionHostedFK()}, + {Name: "cursor_usage_events", Key: []string{"id"}}, + {Name: "starred_sessions", Key: []string{"session_id"}, ForeignKeys: sessionHostedFK()}, + {Name: "excluded_sessions", Key: []string{"id"}}, + {Name: "session_aliases", Key: []string{"session_id", "alias_id"}, ForeignKeys: sessionHostedFK()}, + {Name: "pinned_messages", Key: []string{"id"}, ForeignKeys: sessionHostedFK()}, + {Name: "tool_calls", Key: []string{"id"}, ForeignKeys: sessionHostedFK()}, + {Name: "tool_result_events", Key: []string{"id"}, ForeignKeys: sessionHostedFK()}, + {Name: "secret_findings", Key: []string{"id"}, ForeignKeys: sessionHostedFK()}, + {Name: "insights", Key: []string{"id"}}, + {Name: "model_pricing", Key: []string{"model_pattern"}}, + {Name: "model_pricing_bands", Key: []string{"model_pattern", "above_input_tokens"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"model_pattern"}, Table: "model_pricing", References: []string{"model_pattern"}, Delete: "CASCADE"}}}, + {Name: "genai_pricing", Key: []string{"singleton"}}, + {Name: "source_archives", Key: []string{"source_archive_id"}}, + {Name: "source_project_identity_observations", Key: []string{"source_archive_id", "project", "machine", "root_path", "git_remote"}}, + {Name: "source_project_identity_observation_scopes", Key: []string{"source_archive_id", "project", "machine", "root_path", "git_remote", "publication_scope"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"source_archive_id", "project", "machine", "root_path", "git_remote"}, Table: "source_project_identity_observations", References: []string{"source_archive_id", "project", "machine", "root_path", "git_remote"}, Delete: "CASCADE"}}}, + {Name: "source_session_project_identity_snapshots", Key: []string{"source_archive_id", "source_database_generation", "source_session_id"}}, + {Name: "source_session_project_identity_snapshot_scopes", Key: []string{"source_archive_id", "source_database_generation", "source_session_id", "publication_scope"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"source_archive_id", "source_database_generation", "source_session_id"}, Table: "source_session_project_identity_snapshots", References: []string{"source_archive_id", "source_database_generation", "source_session_id"}, Delete: "CASCADE"}}}, + {Name: "source_worktree_project_mappings", Key: []string{"source_archive_id", "machine", "path_prefix"}}, + {Name: "source_worktree_project_mapping_scopes", Key: []string{"source_archive_id", "machine", "path_prefix", "publication_scope"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"source_archive_id", "machine", "path_prefix"}, Table: "source_worktree_project_mappings", References: []string{"source_archive_id", "machine", "path_prefix"}, Delete: "CASCADE"}}}, + {Name: "raw_devices", Key: []string{"device_id"}}, + {Name: "raw_device_tokens", Key: []string{"token_sha256"}, ForeignKeys: deviceHostedFK()}, + {Name: "raw_upload_sessions", Key: []string{"upload_id"}, ForeignKeys: deviceHostedFK()}, + {Name: "raw_objects", Key: []string{"sha256"}}, + {Name: "raw_manifests", Key: []string{"manifest_id"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"device_id"}, Table: "raw_devices", References: []string{"device_id"}, Delete: "RESTRICT"}, + {Columns: []string{"device_id", "provider", "configured_root_id", "source_key_sha256"}, Table: "raw_source_heads", References: []string{"device_id", "provider", "configured_root_id", "source_key_sha256"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_manifest_entries", Key: []string{"manifest_id", "entry_index"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}}}, + {Name: "raw_manifest_objects", Key: []string{"manifest_id", "entry_index", "object_index"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"manifest_id", "entry_index"}, Table: "raw_manifest_entries", References: []string{"manifest_id", "entry_index"}, Delete: "RESTRICT"}, + {Columns: []string{"sha256", "size_bytes"}, Table: "raw_objects", References: []string{"sha256", "size_bytes"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_source_heads", Key: []string{"device_id", "provider", "configured_root_id", "source_key_sha256"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + {Columns: []string{"device_id"}, Table: "raw_devices", References: []string{"device_id"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_ingest_jobs", Key: []string{"id"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}}}, +}, rawProjectionTables...) + +func sessionHostedFK() []HostedForeignKey { + return []HostedForeignKey{{Columns: []string{"session_id"}, Table: "sessions", References: []string{"id"}, Delete: "CASCADE"}} +} +func deviceHostedFK() []HostedForeignKey { + return []HostedForeignKey{{Columns: []string{"device_id"}, Table: "raw_devices", References: []string{"device_id"}, Delete: "RESTRICT"}} +} +func hostedNames(names []string) (string, error) { + quoted := make([]string, len(names)) + for i, name := range names { + var err error + quoted[i], err = quoteIdentifier(name) + if err != nil { + return "", err + } + } + return strings.Join(quoted, ","), nil +} +func hostedKey(names []string) (string, error) { + return hostedNames(append([]string{"tenant_id"}, names...)) +} +func hostedAction(value string) (string, error) { + switch value { + case "": + return "NO ACTION", nil + case "NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT": + return value, nil + } + return "", fmt.Errorf("invalid hosted referential action") +} + +// InstallHostedTables protects explicitly inventoried tables in an owner-run +// transaction. Call only after creating their ordinary DDL and binding the +// transaction's schema/tenant. It preserves rows and legacy unique targets, +// replaces actual FK constraints, and never rewrites arbitrary DDL text. +func InstallHostedTables(ctx context.Context, tx *sql.Tx, schema, tenant string, tables []HostedTable) error { + if err := validateHostedBinding(schema, tenant); err != nil { + return err + } + if err := checkHostedBinding(ctx, tx, schema, tenant); err != nil { + return err + } + qs, _ := quoteIdentifier(schema) + literal := "'" + strings.ReplaceAll(tenant, "'", "''") + "'" + // Lock all participating tables before inspecting constraints or modifying rows. + for _, table := range tables { + qt, err := quoteIdentifier(table.Name) + if err != nil { + return err + } + if _, err = hostedKey(table.Key); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `LOCK TABLE `+qs+`.`+qt+` IN ACCESS EXCLUSIVE MODE`); err != nil { + return err + } + } + // Remove FKs first so old PK indexes can be replaced. Verify each existing + // relationship against the inventory before removing it (including actions). + for _, table := range tables { + rows, err := tx.QueryContext(ctx, `SELECT c.conname, array_to_string(ARRAY(SELECT a.attname FROM unnest(c.conkey) WITH ORDINALITY k(n,i) JOIN pg_attribute a ON a.attrelid=c.conrelid AND a.attnum=k.n WHERE a.attname<>'tenant_id' ORDER BY k.i),','), r.relname, rn.nspname, array_to_string(ARRAY(SELECT a.attname FROM unnest(c.confkey) WITH ORDINALITY k(n,i) JOIN pg_attribute a ON a.attrelid=c.confrelid AND a.attnum=k.n WHERE a.attname<>'tenant_id' ORDER BY k.i),','), c.confdeltype::text, c.confupdtype::text FROM pg_constraint c JOIN pg_class r ON r.oid=c.confrelid JOIN pg_namespace rn ON rn.oid=r.relnamespace WHERE c.conrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND c.contype='f'`, schema, table.Name) + if err != nil { + return err + } + var drop []string + for rows.Next() { + var name, cols, ref, refSchema, refs, del, upd string + if err = rows.Scan(&name, &cols, &ref, &refSchema, &refs, &del, &upd); err != nil { + rows.Close() + return err + } + found := false + for _, fk := range table.ForeignKeys { + d, _ := hostedAction(fk.Delete) + u, _ := hostedAction(fk.Update) + if refSchema == schema && cols == strings.Join(fk.Columns, ",") && ref == fk.Table && refs == strings.Join(fk.References, ",") && hostedActionCode(d) == del && hostedActionCode(u) == upd { + found = true + break + } + } + if !found { + rows.Close() + return fmt.Errorf("uninventoried foreign key on %s", table.Name) + } + drop = append(drop, name) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + qt, _ := quoteIdentifier(table.Name) + for _, name := range drop { + qn, err := quoteIdentifier(name) + if err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `ALTER TABLE `+qs+`.`+qt+` DROP CONSTRAINT `+qn); err != nil { + return err + } + } + } + for _, table := range tables { + qt, _ := quoteIdentifier(table.Name) + target := qs + `.` + qt + var hasTenant bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND attname='tenant_id' AND NOT attisdropped)`, schema, table.Name).Scan(&hasTenant); err != nil { + return err + } + if !hasTenant { + if _, err := tx.ExecContext(ctx, `ALTER TABLE `+target+` ADD COLUMN tenant_id TEXT NOT NULL DEFAULT `+literal); err != nil { + return err + } + } + var bad bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+target+` WHERE tenant_id IS DISTINCT FROM $1)`, tenant).Scan(&bad); err != nil { + return err + } + if bad { + return fmt.Errorf("existing rows violate hosted tenant binding in %s", table.Name) + } + if _, err := tx.ExecContext(ctx, `ALTER TABLE `+target+` ALTER COLUMN tenant_id SET NOT NULL, ALTER COLUMN tenant_id SET DEFAULT current_setting('agentsview.tenant_id',true)`); err != nil { + return err + } + var pk, keys string + if err := tx.QueryRowContext(ctx, `SELECT c.conname,array_to_string(ARRAY(SELECT a.attname FROM unnest(c.conkey) WITH ORDINALITY k(n,i) JOIN pg_attribute a ON a.attrelid=c.conrelid AND a.attnum=k.n ORDER BY k.i),',') FROM pg_constraint c WHERE c.conrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND c.contype='p'`, schema, table.Name).Scan(&pk, &keys); err != nil { + return err + } + expected := "tenant_id," + strings.Join(table.Key, ",") + if keys != expected { + if keys != strings.Join(table.Key, ",") { + return fmt.Errorf("unexpected hosted primary key on %s", table.Name) + } + old, _ := hostedNames(table.Key) + pkq, err := quoteIdentifier(pk) + if err != nil { + return err + } + // Existing ON CONFLICT clauses still target the original key. + if _, err = tx.ExecContext(ctx, `ALTER TABLE `+target+` ADD UNIQUE (`+old+`), DROP CONSTRAINT `+pkq); err != nil { + return err + } + key, _ := hostedKey(table.Key) + if _, err = tx.ExecContext(ctx, `ALTER TABLE `+target+` ADD PRIMARY KEY (`+key+`)`); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `ALTER TABLE `+target+` ADD CONSTRAINT hosted_tenant_check CHECK (tenant_id = `+literal+`); ALTER TABLE `+target+` ENABLE ROW LEVEL SECURITY; ALTER TABLE `+target+` FORCE ROW LEVEL SECURITY; CREATE POLICY hosted_tenant_policy ON `+target+` USING (tenant_id = current_setting('agentsview.tenant_id',true)) WITH CHECK (tenant_id = current_setting('agentsview.tenant_id',true))`); err != nil { + return err + } + } + for _, table := range tables { + qt, _ := quoteIdentifier(table.Name) + for i, fk := range table.ForeignKeys { + cols, err := hostedKey(fk.Columns) + if err != nil { + return err + } + refs, err := hostedKey(fk.References) + if err != nil { + return err + } + ref, err := quoteIdentifier(fk.Table) + if err != nil { + return err + } + del, err := hostedAction(fk.Delete) + if err != nil { + return err + } + upd, err := hostedAction(fk.Update) + if err != nil { + return err + } + if _, err = tx.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s.%s ADD CONSTRAINT hosted_fk_%d FOREIGN KEY (%s) REFERENCES %s.%s (%s) ON DELETE %s ON UPDATE %s`, qs, qt, i, cols, qs, ref, refs, del, upd)); err != nil { + return err + } + } + } + return nil +} +func hostedActionCode(action string) string { + return map[string]string{"NO ACTION": "a", "RESTRICT": "r", "CASCADE": "c", "SET NULL": "n", "SET DEFAULT": "d"}[action] +} diff --git a/internal/postgres/push.go b/internal/postgres/push.go index bc3a0b365d..a5d33a644f 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -2266,6 +2266,21 @@ func (s *Sync) pushSession( ctx context.Context, tx *sql.Tx, sess db.Session, markerID string, legacyMarkerMachines []string, ) error { + return writePGSession(ctx, tx, sess, markerID, legacyMarkerMachines, pgSessionWriteOptions{ + Machine: s.machine, ArchiveID: s.archiveID, DatabaseGeneration: s.databaseGeneration, UsageOnly: s.local.ArchiveContent().UsageOnly(), + }) +} + +type pgSessionWriteOptions struct { + Machine, ArchiveID string + DatabaseGeneration string + UsageOnly bool + SkipAliases bool +} + +// writePGSession is the shared transaction-owned row kernel. Mirror ownership +// is passed explicitly; it never reads the source archive or global pool. +func writePGSession(ctx context.Context, tx *sql.Tx, sess db.Session, markerID string, legacyMarkerMachines []string, options pgSessionWriteOptions) error { createdAt, ok := ParseSQLiteTimestamp(sess.CreatedAt) if !ok { return fmt.Errorf( @@ -2286,7 +2301,7 @@ func (s *Sync) pushSession( return fmt.Errorf("parsing session %s deleted_at: %w", sess.ID, err) } isAutomated := sess.IsAutomated - pushedMachine := pushedSessionMachine(sess, s.machine) + pushedMachine := pushedSessionMachine(sess, options.Machine) var existingMachine sql.NullString var existingOwnerMarker sql.NullString checkErr := tx.QueryRowContext(ctx, @@ -2588,11 +2603,11 @@ func (s *Sync) pushSession( sanitizePG(sess.AgentLabel), sanitizePG(sess.Entrypoint), sanitizePG(sess.SessionKind), - s.archiveID, - s.databaseGeneration, + options.ArchiveID, + options.DatabaseGeneration, sess.FilePath, string(legacyMarkerMachinesJSON), - s.local.ArchiveContent().UsageOnly(), + options.UsageOnly, ) if err != nil { return err @@ -2638,13 +2653,15 @@ func (s *Sync) pushSession( if excluded { return errSessionExcluded } - if s.local.ArchiveContent().UsageOnly() { + if options.UsageOnly { if err := clearSessionVectorsTx(ctx, tx, sess.ID); err != nil { return err } } - if err := replacePGSessionAliases(ctx, tx, sess); err != nil { - return err + if !options.SkipAliases { + if err := replacePGSessionAliases(ctx, tx, sess); err != nil { + return err + } } return nil } @@ -4135,6 +4152,13 @@ func (s *Sync) pushSecretFindings( return deleted > 0, nil } + if err := bulkInsertSecretFindings(ctx, tx, sessionID, findings); err != nil { + return false, err + } + return true, nil +} + +func bulkInsertSecretFindings(ctx context.Context, tx *sql.Tx, sessionID string, findings []db.SecretFinding) error { const sfBatch = 50 for i := 0; i < len(findings); i += sfBatch { end := min(i+sfBatch, len(findings)) @@ -4171,13 +4195,13 @@ func (s *Sync) pushSecretFindings( if _, err := tx.ExecContext( ctx, b.String(), args..., ); err != nil { - return false, fmt.Errorf( + return fmt.Errorf( "bulk inserting secret_findings for %s: %w", sessionID, err, ) } } - return true, nil + return nil } // normalizeSyncTimestamps ensures schema exists and normalizes diff --git a/internal/postgres/raw_capture_identity_pgtest_test.go b/internal/postgres/raw_capture_identity_pgtest_test.go new file mode 100644 index 0000000000..f3ed71bfbf --- /dev/null +++ b/internal/postgres/raw_capture_identity_pgtest_test.go @@ -0,0 +1,67 @@ +//go:build pgtest + +package postgres + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawtest" +) + +// A filename alias does not replace a missing provider identity. Identical +// real-provider results must coalesce only when that native key is present. +func TestRawClaudeProviderIdentityControlsDeviceCoalescing(t *testing.T) { + for _, known := range []bool{false, true} { + name := "missing_identity" + if known { + name = "native_identity" + } + t.Run(name, func(t *testing.T) { + root := t.TempDir() + path := rawtest.Claude(t, root) + if !known { + body, err := os.ReadFile(path) + require.NoError(t, err) + body = []byte(strings.ReplaceAll(string(body), `,"sessionId":"`+rawtest.ClaudeID+`"`, "")) + require.NoError(t, os.WriteFile(path, body, 0600)) + } + p, ok := parser.NewProvider(parser.AgentClaude, parser.ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := parser.DiscoverRawCaptureSources(t.Context(), p) + require.NoError(t, err) + require.Len(t, sources.Sources, 1) + parsed, err := p.Parse(t.Context(), parser.ParseRequest{Source: sources.Sources[0], ForceParse: true}) + require.NoError(t, err) + require.Len(t, parsed.Results, 1) + expected := "" + if known { + expected = rawtest.ClaudeID + } + assert.Equal(t, expected, parsed.Results[0].Result.Session.SourceSessionID) + f := newProjectionFixture(t) + for _, device := range []string{"device-a", "device-b"} { + m, _ := f.accept(t, device, "capture", "", parser.AgentClaude) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, rawderive.ParsedManifest{Outcome: parsed})) + } + identity, err := f.sink.Resolve(t.Context(), rawtest.ClaudeID) + require.NoError(t, err) + var revisions, groups int + require.NoError(t, f.runtime.QueryRow(`SELECT count(DISTINCT content_revision),count(DISTINCT group_id) FROM raw_content_revisions`).Scan(&revisions, &groups)) + assert.Equal(t, 1, revisions, "content hash remains deterministic across devices") + if known { + assert.Equal(t, RawIdentityUnique, identity.State) + assert.Equal(t, 1, groups) + } else { + assert.Equal(t, RawIdentityAmbiguous, identity.State) + assert.Equal(t, 2, groups) + assert.Len(t, identity.Variants, 2) + } + }) + } +} diff --git a/internal/postgres/raw_derive_pipeline_pgtest_test.go b/internal/postgres/raw_derive_pipeline_pgtest_test.go index 4d0856759b..628120bbce 100644 --- a/internal/postgres/raw_derive_pipeline_pgtest_test.go +++ b/internal/postgres/raw_derive_pipeline_pgtest_test.go @@ -3,14 +3,11 @@ package postgres import ( - "context" - "database/sql" "os" "path/filepath" "testing" "time" - _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/agentsview/internal/artifact" @@ -19,58 +16,41 @@ import ( "go.kenn.io/agentsview/internal/rawcheckpoint" "go.kenn.io/agentsview/internal/rawderive" "go.kenn.io/agentsview/internal/rawsync" + "go.kenn.io/agentsview/internal/rawtest" ) func TestRawCapturedSourcesReachHostedWorker(t *testing.T) { - for _, agent := range []parser.AgentType{parser.AgentClaude, parser.AgentZCode} { + for _, agent := range []parser.AgentType{parser.AgentClaude, parser.AgentZCode, parser.AgentCodex} { t.Run(string(agent), func(t *testing.T) { - pg, metadata := newRawIngestTestStore(t) - identity := rawIngestIdentity(t, "tenant-a") + f := newHostedFixture(t, "tenant-capture") + pg := f.runtime + metadata, err := NewHostedRawIngestStore(pg, f.tenant, "parser-data-17") + require.NoError(t, err) + identity := rawIngestIdentity(t, f.tenant) + _, err = pg.Exec(`INSERT INTO raw_devices(device_id,display_name,credential_sha256,created_at) VALUES($1,'synthetic device',$2,clock_timestamp())`, identity.DeviceID, make([]byte, 32)) + require.NoError(t, err) root := t.TempDir() + ids := []string{rawtest.ClaudeID} if agent == parser.AgentClaude { - contents, err := os.ReadFile("../parser/testdata/claude/valid_session.jsonl") - require.NoError(t, err) - project := filepath.Join(root, "project") - require.NoError(t, os.Mkdir(project, 0o700)) - require.NoError(t, os.WriteFile(filepath.Join(project, "session.jsonl"), contents, 0o600)) + rawtest.Claude(t, root) + rawtest.ClaudeChild(t, root) + ids = append(ids, rawtest.ClaudeChildID) + } else if agent == parser.AgentZCode { + rawtest.ZCode(t, root) + ids = []string{rawtest.ZCodeID, rawtest.ZCodeBillableID} } else { - dbDir := filepath.Join(root, "db") - require.NoError(t, os.Mkdir(dbDir, 0o700)) - sourceDB, err := sql.Open("sqlite3", filepath.Join(dbDir, parser.ZCodeDBName)) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, sourceDB.Close()) }) - _, err = sourceDB.Exec(` - PRAGMA journal_mode = WAL; - CREATE TABLE session ( - id TEXT PRIMARY KEY, project_id TEXT, workspace_id TEXT, - directory TEXT, title TEXT, time_created INTEGER, time_updated INTEGER - ); - CREATE TABLE message ( - id TEXT PRIMARY KEY, session_id TEXT, time_created TEXT, data TEXT - ); - CREATE TABLE part ( - id TEXT PRIMARY KEY, message_id TEXT, session_id TEXT, time_created TEXT, data TEXT - ); - INSERT INTO session VALUES ( - 'session-a', NULL, NULL, '/workspace/project', 'Captured session', - 1783339200000, 1783339260000 - ); - INSERT INTO message VALUES ( - 'message-a', 'session-a', '2026-07-06T12:00:01Z', '{"role":"user"}' - ); - INSERT INTO part VALUES ( - 'part-a', 'message-a', 'session-a', '2026-07-06T12:00:01Z', - '{"type":"text","text":"Fix the login bug"}' - ); - `) - require.NoError(t, err) + rawtest.CodexTools(t, root) + ids = []string{"codex:" + rawtest.CodexToolsID} } + oracle, engine := rawtest.Oracle(t, agent, root, "", "") + require.Equal(t, len(ids), engine.SyncAll(t.Context(), nil).Synced) + provider, ok := parser.NewProvider(agent, parser.ProviderConfig{Roots: []string{root}}) require.True(t, ok) sources, err := parser.DiscoverRawCaptureSources(t.Context(), provider) require.NoError(t, err) require.True(t, sources.Complete) - require.Len(t, sources.Sources, 1) + require.NotEmpty(t, sources.Sources) checkpointDir := t.TempDir() checkpoint, err := rawcheckpoint.OpenWithOptions(t.Context(), filepath.Join(checkpointDir, "checkpoint.db"), rawcheckpoint.Options{ SpoolDir: filepath.Join(checkpointDir, "spool"), MaxOutboxBytes: 1 << 20, @@ -78,12 +58,6 @@ func TestRawCapturedSourcesReachHostedWorker(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, checkpoint.Close()) }) require.NoError(t, checkpoint.SetDevice(t.Context(), identity.DeviceID)) - capture, err := rawcapture.New(checkpoint).Capture(t.Context(), provider, sources.Sources[0]) - require.NoError(t, err) - require.Equal(t, rawcapture.StatusCaptured, capture.Status) - manifest, found, err := checkpoint.FinalizeNextManifest(t.Context(), identity.DeviceID) - require.NoError(t, err) - require.True(t, found) repository, err := artifact.OpenRepository(t.Context(), t.TempDir()) require.NoError(t, err) @@ -93,21 +67,12 @@ func TestRawCapturedSourcesReachHostedWorker(t *testing.T) { limits := rawsync.DefaultManifestLimits() service, err := rawsync.NewService(objects, metadata, limits, "parser-data-17") require.NoError(t, err) - for _, entry := range manifest.Entries { - for _, object := range entry.Objects { - file, err := os.Open(checkpoint.ObjectPath(object)) - require.NoError(t, err) - _, uploadErr := service.FinalizeObject(t.Context(), identity, agent, object, file) - require.NoError(t, file.Close()) - require.NoError(t, uploadErr) - } - } - commit, err := service.CommitManifest(t.Context(), identity, manifest) - require.NoError(t, err) + dispatch, err := rawderive.NewProviderParser(parser.ProviderFactories(), "hosted-worker") require.NoError(t, err) materializationDir := t.TempDir() - sink := &rawParseRecordingSink{store: metadata} + sink, err := NewRawProjectionStore(pg, RawProjectionOptions{Tenant: f.tenant}) + require.NoError(t, err) worker, err := rawderive.NewWorker(rawderive.WorkerConfig{ Queue: metadata, Manifests: rawderive.ManifestLoader{Store: objects, Limits: limits}, Materializer: rawderive.Materializer{Store: objects, BaseDir: materializationDir, MaxTotalBytes: 1 << 20}, @@ -117,42 +82,55 @@ func TestRawCapturedSourcesReachHostedWorker(t *testing.T) { }) require.NoError(t, err) - result, err := worker.RunBatch(t.Context()) + for _, source := range sources.Sources { + capture, err := rawcapture.New(checkpoint).Capture(t.Context(), provider, source) + require.NoError(t, err) + require.Equal(t, rawcapture.StatusCaptured, capture.Status) + manifest, found, err := checkpoint.FinalizeNextManifest(t.Context(), identity.DeviceID) + require.NoError(t, err) + require.True(t, found) + for _, entry := range manifest.Entries { + for _, object := range entry.Objects { + missing, err := service.MissingObjects(t.Context(), identity, agent, []rawsync.ObjectRef{object}) + require.NoError(t, err) + if len(missing) == 0 { + continue + } + file, err := os.Open(checkpoint.ObjectPath(object)) + require.NoError(t, err) + _, uploadErr := service.FinalizeObject(t.Context(), identity, agent, object, file) + require.NoError(t, file.Close()) + require.NoError(t, uploadErr) + } + } + commit, err := service.CommitManifest(t.Context(), identity, manifest) + require.NoError(t, err) + require.NoError(t, checkpoint.BindFinalizedCommit(t.Context(), identity.DeviceID, manifest.CaptureID, commit)) + _, err = checkpoint.AcknowledgeGeneration(t.Context(), identity.DeviceID, manifest.CaptureID, commit) + require.NoError(t, err) + + result, err := worker.RunBatch(t.Context()) + + require.NoError(t, err) + assert.Equal(t, rawderive.BatchResult{Claimed: 1, Succeeded: 1}, result) + var state string + require.NoError(t, pg.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE manifest_id=$1`, commit.ManifestID).Scan(&state)) + assert.Equal(t, "complete", state) + } + + store, err := newHostedAdapter(pg, f.tenant) require.NoError(t, err) - assert.Equal(t, rawderive.BatchResult{Claimed: 1, Succeeded: 1}, result) - require.Len(t, sink.parsed.Outcome.Results, 1) - assert.Empty(t, sink.parsed.Outcome.SourceErrors) - parsed := sink.parsed.Outcome.Results[0].Result - require.NotNil(t, parsed.Session) - assert.Equal(t, agent, parsed.Session.Agent) - require.NotEmpty(t, parsed.Messages) - assert.Equal(t, "Fix the login bug", parsed.Messages[0].Content) - var state string - require.NoError(t, pg.QueryRowContext(t.Context(), - `SELECT state FROM raw_ingest_jobs WHERE manifest_id = $1`, commit.ManifestID, - ).Scan(&state)) - assert.Equal(t, "complete", state) + rawtest.EqualStored(t, t.Context(), oracle, store, ids...) + for _, id := range ids { + resolved, err := sink.Resolve(t.Context(), id) + require.NoError(t, err) + rawtest.EqualUsageEvents(t, t.Context(), oracle, pg, id, resolved.SessionID) + } + remaining, err := os.ReadDir(materializationDir) require.NoError(t, err) assert.Empty(t, remaining, "the worker must remove the captured source tree") }) } } - -// The production projection is a later layer. Record its input while using -// the real lease-completion fence; this test stops at that boundary. -type rawParseRecordingSink struct { - store *RawIngestStore - parsed rawderive.ParsedManifest -} - -func (s *rawParseRecordingSink) Project( - ctx context.Context, lease rawderive.JobLease, _ rawsync.CanonicalManifest, parsed rawderive.ParsedManifest, -) error { - if err := s.store.CompleteRawParseJob(ctx, lease); err != nil { - return err - } - s.parsed = parsed - return nil -} diff --git a/internal/postgres/raw_device_auth_store.go b/internal/postgres/raw_device_auth_store.go index be4cfc5738..5ceb818371 100644 --- a/internal/postgres/raw_device_auth_store.go +++ b/internal/postgres/raw_device_auth_store.go @@ -16,7 +16,8 @@ import ( // RawDeviceAuthStore persists raw-transport devices and short-lived tokens. type RawDeviceAuthStore struct { - db *sql.DB + db *sql.DB + tenant string } // NewRawDeviceAuthStore constructs a PostgreSQL raw device auth store. @@ -27,11 +28,28 @@ func NewRawDeviceAuthStore(db *sql.DB) (*RawDeviceAuthStore, error) { return &RawDeviceAuthStore{db: db}, nil } +// NewTenantRawDeviceAuthStore binds enrollment, credential lookup and tokens to +// the configured tenant before any request credential is authenticated. +func NewTenantRawDeviceAuthStore(database *sql.DB, tenant string) (*RawDeviceAuthStore, error) { + if err := validateHostedBinding("hosted", tenant); err != nil { + return nil, err + } + s, err := NewRawDeviceAuthStore(database) + if err != nil { + return nil, err + } + s.tenant = tenant + return s, nil +} + // EnrollDevice records only the device credential digest. func (s *RawDeviceAuthStore) EnrollDevice( ctx context.Context, record rawsync.DeviceEnrollmentRecord, ) error { + if s.tenant != "" && record.Identity.TenantID != s.tenant { + return rawsync.ErrUnauthorized + } if err := validateRawDeviceEnrollment(record); err != nil { return err } @@ -69,8 +87,8 @@ func (s *RawDeviceAuthStore) AuthenticateCredential( FROM raw_devices WHERE device_id = $1 AND credential_sha256 = $2 - AND revoked_at IS NULL`, - deviceID, credential[:], + AND revoked_at IS NULL AND ($3 = '' OR tenant_id=$3)`, + deviceID, credential[:], s.tenant, ).Scan(&identity.TenantID, &identity.DeviceID) if errors.Is(err, sql.ErrNoRows) { return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized @@ -80,6 +98,9 @@ func (s *RawDeviceAuthStore) AuthenticateCredential( "authenticating raw sync device credential: %w", err, ) } + if s.tenant != "" && identity.TenantID != s.tenant { + return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized + } return identity, nil } @@ -103,7 +124,7 @@ func (s *RawDeviceAuthStore) IssueToken( FROM raw_devices WHERE device_id = $1 AND credential_sha256 = $2 - AND revoked_at IS NULL + AND revoked_at IS NULL AND ($7 = '' OR tenant_id=$7) ) INSERT INTO raw_device_tokens ( token_sha256, tenant_id, device_id, scope_bits, issued_at, expires_at @@ -116,7 +137,7 @@ func (s *RawDeviceAuthStore) IssueToken( token.Digest[:], int16(token.Scopes), token.IssuedAt, - token.ExpiresAt, + token.ExpiresAt, s.tenant, ).Scan(&identity.TenantID, &identity.DeviceID) if errors.Is(err, sql.ErrNoRows) { return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized @@ -129,6 +150,9 @@ func (s *RawDeviceAuthStore) IssueToken( } return rawsync.AuthIdentity{}, fmt.Errorf("issuing raw sync token: %w", err) } + if s.tenant != "" && identity.TenantID != s.tenant { + return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized + } return identity, nil } @@ -153,9 +177,9 @@ func (s *RawDeviceAuthStore) AuthenticateToken( AND devices.device_id = tokens.device_id WHERE tokens.token_sha256 = $1 AND tokens.expires_at > $2 - AND devices.revoked_at IS NULL + AND devices.revoked_at IS NULL AND ($4 = '' OR devices.tenant_id=$4) AND (tokens.scope_bits & $3) = $3`, - digest[:], now.UTC(), int16(required), + digest[:], now.UTC(), int16(required), s.tenant, ).Scan(&identity.TenantID, &identity.DeviceID) if errors.Is(err, sql.ErrNoRows) { return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized @@ -163,6 +187,9 @@ func (s *RawDeviceAuthStore) AuthenticateToken( if err != nil { return rawsync.AuthIdentity{}, fmt.Errorf("authenticating raw sync token: %w", err) } + if s.tenant != "" && identity.TenantID != s.tenant { + return rawsync.AuthIdentity{}, rawsync.ErrUnauthorized + } return identity, nil } @@ -172,6 +199,9 @@ func (s *RawDeviceAuthStore) RevokeDevice( identity rawsync.AuthIdentity, revokedAt time.Time, ) (bool, error) { + if s.tenant != "" && identity.TenantID != s.tenant { + return false, rawsync.ErrUnauthorized + } if err := validateRawDeviceAuthIdentity(identity); err != nil { return false, err } diff --git a/internal/postgres/raw_ingest_schema.go b/internal/postgres/raw_ingest_schema.go index f2094930a1..d26856816e 100644 --- a/internal/postgres/raw_ingest_schema.go +++ b/internal/postgres/raw_ingest_schema.go @@ -163,6 +163,8 @@ CREATE TABLE IF NOT EXISTS raw_ingest_jobs ( manifest_id TEXT NOT NULL, stage TEXT NOT NULL CHECK (stage IN ('parse')), processing_version TEXT NOT NULL, + projection_generation BIGINT NOT NULL DEFAULT 0, + projection_selected BOOLEAN NOT NULL DEFAULT true, state TEXT NOT NULL DEFAULT 'ready' CHECK ( state IN ( @@ -325,6 +327,9 @@ func ensureRawIngestSchemaPG(ctx context.Context, db *sql.DB) error { if _, err := db.ExecContext(ctx, rawIngestDDL); err != nil { return fmt.Errorf("creating raw ingest schema: %w", err) } + if err := ensureRawProjectionJobColumns(ctx, db); err != nil { + return err + } if _, err := db.ExecContext(ctx, rawIngestAppendOnlyDDL); err != nil { if !rawIngestAppendOnlyUnsupported(err) { return fmt.Errorf("installing raw ingest append-only guards: %w", err) @@ -337,6 +342,24 @@ func ensureRawIngestSchemaPG(ctx context.Context, db *sql.DB) error { return nil } +// The raw bootstrap also runs on the compatible mirror fast path. Probe before +// taking a migration lock so an already-provisioned schema performs no ALTER. +func ensureRawProjectionJobColumns(ctx context.Context, q interface { + hostedQuerier + ExecContext(context.Context, string, ...any) (sql.Result, error) +}) error { + var current bool + err := q.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid='raw_ingest_jobs'::regclass AND attnum>0 AND NOT attisdropped AND attname IN ('projection_generation','projection_selected') GROUP BY attrelid HAVING count(*)=2)`).Scan(¤t) + if err != nil { + return fmt.Errorf("probing raw projection job columns: %w", err) + } + if current { + return nil + } + _, err = q.ExecContext(ctx, `ALTER TABLE raw_ingest_jobs ADD COLUMN IF NOT EXISTS projection_generation BIGINT NOT NULL DEFAULT 0, ADD COLUMN IF NOT EXISTS projection_selected BOOLEAN NOT NULL DEFAULT true`) + return err +} + func rawIngestAppendOnlyUnsupported(err error) bool { if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { return pgErr.Code == "0A000" diff --git a/internal/postgres/raw_ingest_schema_pgtest_test.go b/internal/postgres/raw_ingest_schema_pgtest_test.go index 9fbdf00bb6..e21f5b38bb 100644 --- a/internal/postgres/raw_ingest_schema_pgtest_test.go +++ b/internal/postgres/raw_ingest_schema_pgtest_test.go @@ -241,6 +241,15 @@ func TestSyncEnsureSchemaCreatesRawCustodyOnLegacyFastPath(t *testing.T) { )`, schemaTestSchema).Scan(&exists)) assert.True(t, exists, "schema-current sync must still install newly introduced custody tables") + // An older raw-only installation also needs the additive job migration, + // without forcing the unrelated mirror schema down its full DDL path. + _, err = pg.ExecContext(t.Context(), `ALTER TABLE raw_ingest_jobs DROP COLUMN projection_generation, DROP COLUMN projection_selected`) + require.NoError(t, err) + syncer = &Sync{pg: pg, schema: schemaTestSchema} + require.NoError(t, syncer.EnsureSchema(t.Context())) + rows, err := pg.QueryContext(t.Context(), `SELECT projection_generation,projection_selected FROM raw_ingest_jobs LIMIT 0`) + require.NoError(t, err) + require.NoError(t, rows.Close()) } // TestSyncEnsureSchemaFastPathToleratesRestrictedRole pins the restricted-role diff --git a/internal/postgres/raw_ingest_store.go b/internal/postgres/raw_ingest_store.go index 3d6f2099a6..434b1dc524 100644 --- a/internal/postgres/raw_ingest_store.go +++ b/internal/postgres/raw_ingest_store.go @@ -20,8 +20,10 @@ const rawIngestBatchRows = 256 // RawIngestStore implements raw custody metadata over PostgreSQL. type RawIngestStore struct { - db *sql.DB - newReceipt func() (string, error) + db *sql.DB + tenant string + hostedVersion string + newReceipt func() (string, error) } // NewRawIngestStore constructs a PostgreSQL raw custody metadata store. @@ -32,6 +34,40 @@ func NewRawIngestStore(db *sql.DB) (*RawIngestStore, error) { return &RawIngestStore{db: db, newReceipt: generateRawIngestReceipt}, nil } +// NewTenantRawIngestStore binds custody and job operations to one configured +// tenant. Callers must separately validate the hosted pool at startup. +func NewTenantRawIngestStore(database *sql.DB, tenant string) (*RawIngestStore, error) { + if err := validateHostedBinding("hosted", tenant); err != nil { + return nil, err + } + s, err := NewRawIngestStore(database) + if err != nil { + return nil, err + } + s.tenant = tenant + return s, nil +} + +// NewHostedRawIngestStore atomically selects the configured projection on new +// acceptance and limits claims to that exact processing version. +func NewHostedRawIngestStore(database *sql.DB, tenant, version string) (*RawIngestStore, error) { + if err := validateRawIngestProcessingVersion(version); err != nil { + return nil, err + } + s, err := NewTenantRawIngestStore(database, tenant) + if err != nil { + return nil, err + } + s.hostedVersion = version + return s, nil +} +func (s *RawIngestStore) validateIdentity(identity rawsync.AuthIdentity) error { + if s.tenant != "" && identity.TenantID != s.tenant { + return rawsync.ErrUnauthorized + } + return validateRawIngestIdentity(identity) +} + // RecordVerifiedObject records an object only after physical verification. func (s *RawIngestStore) RecordVerifiedObject( ctx context.Context, @@ -47,7 +83,7 @@ func (s *RawIngestStore) RecordVerifiedObjects( identity rawsync.AuthIdentity, objects []rawsync.ObjectRef, ) error { - if err := validateRawIngestIdentity(identity); err != nil { + if err := s.validateIdentity(identity); err != nil { return err } unique, err := uniqueRawIngestObjects(objects) @@ -94,7 +130,7 @@ func (s *RawIngestStore) MissingObjects( identity rawsync.AuthIdentity, objects []rawsync.ObjectRef, ) ([]rawsync.ObjectRef, error) { - if err := validateRawIngestIdentity(identity); err != nil { + if err := s.validateIdentity(identity); err != nil { return nil, err } unique, err := uniqueRawIngestObjects(objects) @@ -124,6 +160,12 @@ func (s *RawIngestStore) CommitManifest( manifest rawsync.CanonicalManifest, processingVersion string, ) (rawsync.CommitResult, error) { + if s.hostedVersion != "" && processingVersion != s.hostedVersion { + return rawsync.CommitResult{}, rawsync.ErrInvalid + } + if err := s.validateIdentity(manifest.Identity); err != nil { + return rawsync.CommitResult{}, err + } if err := validateRawIngestProcessingVersion(processingVersion); err != nil { return rawsync.CommitResult{}, err } @@ -250,6 +292,11 @@ func (s *RawIngestStore) CommitManifest( if affected != 1 { return rawsync.CommitResult{}, fmt.Errorf("advancing raw source head affected %d rows", affected) } + if s.hostedVersion != "" { + if _, err := selectRawSourceGenerationTx(ctx, tx, manifest, processingVersion); err != nil { + return rawsync.CommitResult{}, err + } + } if head.ManifestID != "" { // The unique key permits legacy rows with several processing versions // per manifest, so supersede exactly one deterministic candidate per diff --git a/internal/postgres/raw_parse_jobs.go b/internal/postgres/raw_parse_jobs.go index 3f24c171ba..69e684bd43 100644 --- a/internal/postgres/raw_parse_jobs.go +++ b/internal/postgres/raw_parse_jobs.go @@ -43,53 +43,8 @@ func (s *RawIngestStore) ClaimRawParseJobs( } }() - rows, err := tx.QueryContext(ctx, ` - WITH candidates AS ( - SELECT job.id - FROM raw_ingest_jobs AS job - JOIN raw_manifests AS manifest - ON manifest.tenant_id = job.tenant_id - AND manifest.manifest_id = job.manifest_id - WHERE ( - (job.state IN ('ready', 'retrying') AND job.available_at <= now()) - OR (job.state = 'leased' AND job.lease_expires_at <= now()) - ) - AND job.stage = 'parse' - AND EXISTS ( - SELECT 1 - FROM raw_source_heads AS head - WHERE head.tenant_id = manifest.tenant_id - AND head.device_id = manifest.device_id - AND head.provider = manifest.provider - AND head.configured_root_id = manifest.configured_root_id - AND head.source_key_sha256 = manifest.source_key_sha256 - AND head.manifest_id = manifest.manifest_id - ) - ORDER BY job.available_at, job.id - LIMIT $1 - FOR UPDATE OF job SKIP LOCKED - ), claimed AS ( - UPDATE raw_ingest_jobs AS job - SET state = 'leased', attempt_count = job.attempt_count + 1, - lease_owner = $2, - lease_expires_at = now() + ($3 * interval '1 microsecond'), - last_error_class = '', last_error = '', updated_at = now() - FROM candidates - WHERE job.id = candidates.id - RETURNING job.id, job.tenant_id, job.manifest_id, - job.processing_version, job.attempt_count, job.lease_owner, - job.lease_expires_at - ) - SELECT claimed.id, claimed.tenant_id, manifest.device_id, - claimed.manifest_id, claimed.processing_version, - claimed.attempt_count, claimed.lease_owner, claimed.lease_expires_at - FROM claimed - JOIN raw_manifests AS manifest - ON manifest.tenant_id = claimed.tenant_id - AND manifest.manifest_id = claimed.manifest_id - ORDER BY claimed.id`, - limit, owner, leaseDuration.Microseconds(), - ) + query, args := s.rawParseClaimStatement(owner, limit, leaseDuration) + rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return nil, fmt.Errorf("claiming raw parse jobs: %w", err) } @@ -102,6 +57,7 @@ func (s *RawIngestStore) ClaimRawParseJobs( &lease.Identity.DeviceID, &lease.ManifestID, &lease.ProcessingVersion, + &lease.ProjectionGeneration, &lease.Attempt, &lease.Owner, &lease.ExpiresAt, @@ -121,9 +77,9 @@ func (s *RawIngestStore) ClaimRawParseJobs( // supersession only reclaims rows no claim could ever lease. Running it // after a short claim keeps cleanup bounded without charging the common // full-batch path for an O(backlog) head-probe scan. - if len(leasing) < limit { - if err := supersedeObsoleteRawParseJobs( - ctx, tx, maxRawParseSupersedeBatch, + if len(leasing) < limit && s.hostedVersion == "" { + if err := supersedeTenantObsoleteRawParseJobs( + ctx, tx, maxRawParseSupersedeBatch, s.tenant, ); err != nil { return nil, err } @@ -155,7 +111,7 @@ func supersedeObsoleteRawParseJobs( } // Future retries stay outside the idle cleanup scan until they become due. -const rawParseSupersedeSQL = ` +const rawParseSupersedePrefix = ` UPDATE raw_ingest_jobs AS job SET state = 'superseded', lease_owner = '', lease_expires_at = NULL, updated_at = now() @@ -169,7 +125,8 @@ const rawParseSupersedeSQL = ` (obsolete.state IN ('ready', 'retrying') AND obsolete.available_at <= now()) OR (obsolete.state = 'leased' AND obsolete.lease_expires_at <= now()) ) - AND obsolete.stage = 'parse' + AND obsolete.stage = 'parse'` +const rawParseSupersedeSuffix = ` AND NOT EXISTS ( SELECT 1 FROM raw_source_heads AS head @@ -185,12 +142,29 @@ const rawParseSupersedeSQL = ` FOR UPDATE OF obsolete SKIP LOCKED )` +const rawParseSupersedeSQL = rawParseSupersedePrefix + rawParseSupersedeSuffix +const rawParseSupersedeTenantSQL = rawParseSupersedePrefix + ` AND obsolete.tenant_id = $2 ` + rawParseSupersedeSuffix + ` AND job.tenant_id = $2 ` + +func supersedeTenantObsoleteRawParseJobs(ctx context.Context, tx *sql.Tx, limit int, tenant string) error { + if tenant == "" { + return supersedeObsoleteRawParseJobs(ctx, tx, limit) + } + _, err := tx.ExecContext(ctx, rawParseSupersedeTenantSQL, limit, tenant) + if err != nil { + return fmt.Errorf("superseding tenant raw parse jobs: %w", err) + } + return nil +} + // HeartbeatRawParseJob extends an active lease held by the current source head. func (s *RawIngestStore) HeartbeatRawParseJob( ctx context.Context, lease rawderive.JobLease, leaseDuration time.Duration, ) error { + if s.tenant != "" && lease.Identity.TenantID != s.tenant { + return rawderive.ErrLeaseLost + } if err := validateRawParseLease(lease); err != nil { return err } @@ -202,6 +176,7 @@ func (s *RawIngestStore) HeartbeatRawParseJob( SET lease_expires_at = now() + ($4 * interval '1 microsecond'), updated_at = now() WHERE job.id = $1 AND job.lease_owner = $2 AND job.attempt_count = $3 + AND ($5 = '' OR (job.tenant_id = $5 AND job.manifest_id = $6 AND job.processing_version = $7)) AND job.state = 'leased' AND job.lease_expires_at > now() AND EXISTS ( SELECT 1 @@ -216,7 +191,7 @@ func (s *RawIngestStore) HeartbeatRawParseJob( WHERE manifest.tenant_id = job.tenant_id AND manifest.manifest_id = job.manifest_id )`, - lease.ID, lease.Owner, lease.Attempt, leaseDuration.Microseconds(), + lease.ID, lease.Owner, lease.Attempt, leaseDuration.Microseconds(), s.tenant, lease.ManifestID, lease.ProcessingVersion, ) if err != nil { return fmt.Errorf("heartbeating raw parse job: %w", err) @@ -229,6 +204,9 @@ func (s *RawIngestStore) CompleteRawParseJob( ctx context.Context, lease rawderive.JobLease, ) error { + if s.tenant != "" && lease.Identity.TenantID != s.tenant { + return rawderive.ErrLeaseLost + } if err := validateRawParseLease(lease); err != nil { return err } @@ -237,6 +215,7 @@ func (s *RawIngestStore) CompleteRawParseJob( SET state = 'complete', lease_owner = '', lease_expires_at = NULL, last_error_class = '', last_error = '', updated_at = now() WHERE job.id = $1 AND job.lease_owner = $2 AND job.attempt_count = $3 + AND ($4 = '' OR (job.tenant_id = $4 AND job.manifest_id = $5 AND job.processing_version = $6)) AND job.state = 'leased' AND job.lease_expires_at > now() AND EXISTS ( SELECT 1 @@ -251,7 +230,7 @@ func (s *RawIngestStore) CompleteRawParseJob( WHERE manifest.tenant_id = job.tenant_id AND manifest.manifest_id = job.manifest_id )`, - lease.ID, lease.Owner, lease.Attempt, + lease.ID, lease.Owner, lease.Attempt, s.tenant, lease.ManifestID, lease.ProcessingVersion, ) if err != nil { return fmt.Errorf("completing raw parse job: %w", err) @@ -267,6 +246,9 @@ func (s *RawIngestStore) RetryRawParseJob( errorClass string, message string, ) error { + if s.tenant != "" && lease.Identity.TenantID != s.tenant { + return rawderive.ErrLeaseLost + } if err := validateRawParseLease(lease); err != nil { return err } @@ -279,6 +261,7 @@ func (s *RawIngestStore) RetryRawParseJob( lease_expires_at = NULL, last_error_class = $5, last_error = $6, updated_at = now() WHERE job.id = $1 AND job.lease_owner = $2 AND job.attempt_count = $3 + AND ($7 = '' OR (job.tenant_id = $7 AND job.manifest_id = $8 AND job.processing_version = $9)) AND job.state = 'leased' AND job.lease_expires_at > now() AND EXISTS ( SELECT 1 @@ -293,7 +276,7 @@ func (s *RawIngestStore) RetryRawParseJob( WHERE manifest.tenant_id = job.tenant_id AND manifest.manifest_id = job.manifest_id )`, - lease.ID, lease.Owner, lease.Attempt, availableAt.UTC(), errorClass, message, + lease.ID, lease.Owner, lease.Attempt, availableAt.UTC(), errorClass, message, s.tenant, lease.ManifestID, lease.ProcessingVersion, ) if err != nil { return fmt.Errorf("retrying raw parse job: %w", err) @@ -309,6 +292,9 @@ func (s *RawIngestStore) FailRawParseJob( errorClass string, message string, ) error { + if s.tenant != "" && lease.Identity.TenantID != s.tenant { + return rawderive.ErrLeaseLost + } if err := validateRawParseLease(lease); err != nil { return err } @@ -317,6 +303,7 @@ func (s *RawIngestStore) FailRawParseJob( SET state = 'failed', lease_owner = '', lease_expires_at = NULL, last_error_class = $4, last_error = $5, updated_at = now() WHERE job.id = $1 AND job.lease_owner = $2 AND job.attempt_count = $3 + AND ($6 = '' OR (job.tenant_id = $6 AND job.manifest_id = $7 AND job.processing_version = $8)) AND job.state = 'leased' AND job.lease_expires_at > now() AND EXISTS ( SELECT 1 @@ -331,7 +318,7 @@ func (s *RawIngestStore) FailRawParseJob( WHERE manifest.tenant_id = job.tenant_id AND manifest.manifest_id = job.manifest_id )`, - lease.ID, lease.Owner, lease.Attempt, errorClass, message, + lease.ID, lease.Owner, lease.Attempt, errorClass, message, s.tenant, lease.ManifestID, lease.ProcessingVersion, ) if err != nil { return fmt.Errorf("failing raw parse job: %w", err) @@ -374,3 +361,61 @@ func requireRawParseLeaseUpdate(result sql.Result) error { } return nil } + +func (s *RawIngestStore) rawParseClaimStatement(owner string, limit int, leaseDuration time.Duration) (string, []any) { + query := ` + WITH candidates AS ( + SELECT job.id + FROM raw_ingest_jobs AS job + JOIN raw_manifests AS manifest + ON manifest.tenant_id = job.tenant_id + AND manifest.manifest_id = job.manifest_id + WHERE ( + (job.state IN ('ready', 'retrying') AND job.available_at <= now()) + OR (job.state = 'leased' AND job.lease_expires_at <= now()) + ) + AND job.stage = 'parse' AND job.projection_selected + AND ($4 = '' OR job.tenant_id = $4) + {{hosted}} + AND EXISTS ( + SELECT 1 + FROM raw_source_heads AS head + WHERE head.tenant_id = manifest.tenant_id + AND head.device_id = manifest.device_id + AND head.provider = manifest.provider + AND head.configured_root_id = manifest.configured_root_id + AND head.source_key_sha256 = manifest.source_key_sha256 + AND head.manifest_id = manifest.manifest_id + ) + ORDER BY job.available_at, job.id + LIMIT $1 + FOR UPDATE OF job SKIP LOCKED + ), claimed AS ( + UPDATE raw_ingest_jobs AS job + SET state = 'leased', attempt_count = job.attempt_count + 1, + lease_owner = $2, + lease_expires_at = now() + ($3 * interval '1 microsecond'), + last_error_class = '', last_error = '', updated_at = now() + FROM candidates + WHERE job.id = candidates.id AND ($4 = '' OR job.tenant_id = $4) + RETURNING job.id, job.tenant_id, job.manifest_id, + job.processing_version, job.projection_generation, job.attempt_count, job.lease_owner, + job.lease_expires_at + ) + SELECT claimed.id, claimed.tenant_id, manifest.device_id, + claimed.manifest_id, claimed.processing_version, claimed.projection_generation, + claimed.attempt_count, claimed.lease_owner, claimed.lease_expires_at + FROM claimed + JOIN raw_manifests AS manifest + ON manifest.tenant_id = claimed.tenant_id + AND manifest.manifest_id = claimed.manifest_id + ORDER BY claimed.id` + args := []any{limit, owner, leaseDuration.Microseconds(), s.tenant} + predicate := "" + if s.hostedVersion != "" { + predicate = `AND job.processing_version=$5 AND job.projection_generation>0 AND EXISTS (SELECT 1 FROM raw_source_projections selected WHERE selected.tenant_id=job.tenant_id AND selected.selected_job_id=job.id AND selected.selected_manifest_id=job.manifest_id AND selected.processing_version=job.processing_version AND selected.projection_generation=job.projection_generation)` + args = append(args, s.hostedVersion) + query = strings.ReplaceAll(query, "($4 = '' OR job.tenant_id = $4)", "job.tenant_id = $4") + } + return strings.Replace(query, "{{hosted}}", predicate, 1), args +} diff --git a/internal/postgres/raw_projection.go b/internal/postgres/raw_projection.go new file mode 100644 index 0000000000..0364ca206c --- /dev/null +++ b/internal/postgres/raw_projection.go @@ -0,0 +1,322 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "slices" + + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +type rawBranch struct { + ID, Source, Group, Member, Session, Revision, Manifest, Version string + Generation int64 + Active bool + Payload []byte +} + +func loadRawBranches(ctx context.Context, q hostedQuerier, where string, arg string) ([]rawBranch, error) { + rows, err := q.QueryContext(ctx, `SELECT branch_id,source_id,group_id,member_id,session_id,content_revision,manifest_id,processing_version,projection_generation,active,prior_payload FROM raw_session_branches WHERE `+where+`=$1 ORDER BY branch_id`, arg) + if err != nil { + return nil, err + } + defer rows.Close() + var branches []rawBranch + for rows.Next() { + var b rawBranch + if err = rows.Scan(&b.ID, &b.Source, &b.Group, &b.Member, &b.Session, &b.Revision, &b.Manifest, &b.Version, &b.Generation, &b.Active, &b.Payload); err != nil { + return nil, err + } + branches = append(branches, b) + } + return branches, rows.Err() +} + +func (s *RawProjectionStore) Project(ctx context.Context, lease rawderive.JobLease, m rawsync.CanonicalManifest, parsed rawderive.ParsedManifest) error { + if err := s.validateManifest(m); err != nil { + return err + } + if parsed.Tombstone != (m.Manifest.Kind == rawsync.ManifestTombstone) { + return fmt.Errorf("raw projection tombstone mismatch") + } + if parsed.Tombstone && len(parsed.Outcome.Results) > 0 { + return fmt.Errorf("tombstone cannot publish members") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if err = s.lockHead(ctx, tx, m); err != nil { + return err + } + if err = s.lockLease(ctx, tx, m, lease); err != nil { + return err + } + source := rawSourceID(m) + prior, err := loadRawBranches(ctx, tx, "source_id", source) + if err != nil { + return err + } + groups := map[string]string{} + keys := map[string]string{} + priorByGroup := map[string]rawBranch{} + for _, b := range prior { + groups[b.Group] = "" + priorByGroup[b.Group] = b + } + candidates := map[string]ingest.Candidate{} + for _, r := range parsed.Outcome.Results { + c, err := ingest.PrepareCandidate(ctx, r.Result, s.options.Content) + if err != nil { + return err + } + if slices.Contains(parsed.Outcome.ExcludedSessionIDs, c.Session.ID) { + continue + } + if c.Session.ID == "" || c.Session.Agent == "" { + return fmt.Errorf("raw projection member identity missing") + } + group, key := rawGroupID(m, c.Session) + if _, duplicate := candidates[group]; duplicate { + return fmt.Errorf("duplicate raw projection member") + } + groups[group] = c.Session.ID + keys[group] = key + candidates[group] = c + } + ordered := make([]string, 0, len(groups)) + for group := range groups { + ordered = append(ordered, group) + } + slices.Sort(ordered) + // Lock known and newly introduced public aliases before sorted group locks. + // This includes first publication, where legacy validation cannot lock a row. + var aliases []string + rows, err := tx.QueryContext(ctx, `SELECT alias_id FROM raw_session_public_aliases WHERE group_id=ANY($1) ORDER BY alias_id`, ordered) + if err != nil { + return err + } + for rows.Next() { + var alias string + if err = rows.Scan(&alias); err != nil { + rows.Close() + return err + } + aliases = append(aliases, alias) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for group, base := range groups { + if base != "" { + aliases = append(aliases, base, base+"~"+rawDigest("branch-v1", group, source)) + aliases = append(aliases, pgSessionAliasIDs(candidates[group].Session)...) + } + } + if err = lockHostedAliases(ctx, tx, aliases); err != nil { + return err + } + for _, group := range ordered { + if base := groups[group]; base != "" { + _, err = tx.ExecContext(ctx, `INSERT INTO raw_session_groups(group_id,provider,logical_key,base_alias) VALUES($1,$2,$3,$4) ON CONFLICT(group_id) DO NOTHING`, group, candidates[group].Session.Agent, keys[group], base) + if err != nil { + return err + } + } + var storedKey string + err = tx.QueryRowContext(ctx, `SELECT logical_key FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, group).Scan(&storedKey) + if err != nil { + return err + } + if key, ok := keys[group]; ok && key != storedKey { + return fmt.Errorf("raw logical identity digest collision") + } + } + complete := parsed.Tombstone || parsed.Outcome.ResultSetComplete && len(parsed.Outcome.SourceErrors) == 0 + for _, r := range parsed.Outcome.Results { + if r.DataVersion == parser.DataVersionNeedsRetry || r.RetryReason != "" { + complete = false + } + } + changed := false + derivedChanged := false + for _, group := range ordered { + c, ok := candidates[group] + if !ok { + continue + } + previous, exists := priorByGroup[group] + var old ingest.PreparedSession + var history *ingest.PriorSession + if exists { + old, err = loadRawPayload(ctx, tx, previous.Session) + if err != nil { + return err + } + metadata, decodeErr := decodeRawPayload(previous.Payload) + if decodeErr != nil { + return decodeErr + } + old.Session = metadata.Session + history = &ingest.PriorSession{Session: old.Session, Messages: old.Messages} + } + decision, err := ingest.ReconcileProviderHistory(ctx, c, history, s.options.Content) + if err != nil { + return err + } + var prepared ingest.PreparedSession + if decision.Action == ingest.HistoryPreserve { + prepared = old + } else { + prepared, err = ingest.Finalize(ctx, decision.Candidate, s.options.Content) + if err != nil { + return err + } + } + prepared.Signals = ingest.RefreshSignalRecencyAt(prepared.Session, prepared.Messages, prepared.Signals, s.options.Now()) + revision, err := rawContentRevision(prepared) + if err != nil { + return err + } + sessionID := "raw-row-" + rawDigest("row-v1", group, revision) + branchID := rawDigest("branch-v1", group, source) + payload, err := encodeRawPayload(prepared) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_content_revisions(session_id,group_id,content_revision,payload) VALUES($1,$2,$3,$4) ON CONFLICT(session_id) DO NOTHING`, sessionID, group, revision, payload) + if err != nil { + return err + } + visibleChange, err := publishRawRecency(ctx, tx, sessionID, prepared.Signals) + if err != nil { + return err + } + derivedChanged = derivedChanged || visibleChange + + metadata, err := encodeRawPayload(ingest.PreparedSession{Session: prepared.Session}) + if err != nil { + return err + } + observed, err := encodeRawPayload(ingest.PreparedSession{Session: c.Session}) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_session_branches(branch_id,source_id,group_id,member_id,session_id,content_revision,manifest_id,processing_version,projection_generation,active,prior_payload) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,true,$10) + ON CONFLICT(branch_id) DO UPDATE SET session_id=EXCLUDED.session_id,content_revision=EXCLUDED.content_revision,manifest_id=EXCLUDED.manifest_id,processing_version=EXCLUDED.processing_version,projection_generation=EXCLUDED.projection_generation,active=true,prior_payload=EXCLUDED.prior_payload`, branchID, source, group, c.Session.ID, sessionID, revision, m.ManifestID, lease.ProcessingVersion, lease.ProjectionGeneration, metadata) + if err != nil { + return err + } + if err = writeRawLinks(ctx, tx, branchID, prepared); err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_source_contributions(branch_id,manifest_id,payload,projection_generation,processing_version,prior_contributed) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(branch_id,manifest_id,projection_generation) DO NOTHING`, branchID, m.ManifestID, observed, lease.ProjectionGeneration, lease.ProcessingVersion, decision.PriorContributed) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_session_public_aliases(alias_id,group_id,anchor_branch) VALUES($1,$2,''),($3,$2,$4) ON CONFLICT(alias_id,group_id) DO NOTHING`, groups[group], group, groups[group]+"~"+branchID, branchID) + if err != nil { + return err + } + + for _, alias := range pgSessionAliasIDs(c.Session) { + _, err = tx.ExecContext(ctx, `INSERT INTO raw_session_public_aliases(alias_id,group_id,anchor_branch) VALUES($1,$2,'') ON CONFLICT(alias_id,group_id) DO NOTHING`, alias, group) + if err != nil { + return err + } + } + changed = changed || !exists || !previous.Active || previous.Session != sessionID + } + for _, b := range prior { + _, present := candidates[b.Group] + excluded := slices.Contains(parsed.Outcome.ExcludedSessionIDs, b.Member) + if b.Active && (excluded || !present && complete) { + _, err = tx.ExecContext(ctx, `UPDATE raw_session_branches SET active=false WHERE branch_id=$1`, b.ID) + if err != nil { + return err + } + changed = true + } + } + if changed || derivedChanged { + var revision int64 + err = tx.QueryRowContext(ctx, `INSERT INTO raw_corpus_state(singleton,corpus_revision,identity_revision) VALUES(1,1,CASE WHEN $1 THEN 1 ELSE 0 END) ON CONFLICT(singleton) DO UPDATE SET corpus_revision=raw_corpus_state.corpus_revision+1,identity_revision=raw_corpus_state.identity_revision+EXCLUDED.identity_revision RETURNING corpus_revision`, changed).Scan(&revision) + if err != nil { + return err + } + if changed { + for _, group := range ordered { + if err = s.materializeGroup(ctx, tx, group, revision); err != nil { + return err + } + } + } + } + _, err = tx.ExecContext(ctx, rawSourceProofDeleteSQL, source) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO session_sources(branch_id,group_id,source_id,session_id,physical_session_id,manifest_id,content_revision,processing_version,projection_generation) SELECT b.branch_id,b.group_id,b.source_id,b.session_id,s.id,b.manifest_id,b.content_revision,b.processing_version,b.projection_generation FROM raw_session_branches b LEFT JOIN sessions s ON s.id=b.session_id WHERE b.source_id=$1 AND b.active`, source) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, `UPDATE raw_source_projections SET last_attempt_manifest_id=$2,successful_manifest_id=CASE WHEN $5 THEN $2 ELSE successful_manifest_id END,membership_complete=$3,diagnostics=$4 WHERE source_id=$1`, source, m.ManifestID, complete, fmt.Sprintf("results=%d errors=%d complete=%t", len(parsed.Outcome.Results), len(parsed.Outcome.SourceErrors), complete), complete || len(candidates) > 0) + if err != nil { + return err + } + outcome, err := completeProjectionJob(ctx, tx, lease, complete, s.options.RetryPolicy) + if err != nil { + return err + } + if err = tx.Commit(); err != nil { + return err + } + if outcome != 0 { + return outcome + } + return nil +} + +// loadRawPayload is transaction-bound and does not use alias resolution. +func loadRawPayload(ctx context.Context, tx *sql.Tx, id string) (ingest.PreparedSession, error) { + var payload, recency []byte + err := tx.QueryRowContext(ctx, `SELECT payload,recency_state FROM raw_content_revisions WHERE session_id=$1`, id).Scan(&payload, &recency) + if err != nil { + return ingest.PreparedSession{}, err + } + p, err := decodeRawPayload(payload) + if err != nil { + return p, err + } + if string(recency) != "{}" { + state, err := decodeRawRecency(recency) + if err != nil { + return p, err + } + state.apply(&p.Signals) + } + return p, nil +} +func rawPhysicalExists(ctx context.Context, tx *sql.Tx, id string) (bool, error) { + var kind string + err := tx.QueryRowContext(ctx, `SELECT provenance_kind FROM sessions WHERE id=$1`, id).Scan(&kind) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + if kind != "raw" { + return false, fmt.Errorf("raw physical identity conflicts with legacy proof") + } + return true, nil +} diff --git a/internal/postgres/raw_projection_cohorts_pgtest_test.go b/internal/postgres/raw_projection_cohorts_pgtest_test.go new file mode 100644 index 0000000000..3ff974ea65 --- /dev/null +++ b/internal/postgres/raw_projection_cohorts_pgtest_test.go @@ -0,0 +1,95 @@ +//go:build pgtest + +package postgres + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/rawsync" + "slices" + "testing" +) + +func TestRawProjectionThreeSourceCohortNamesAndPinNotes(t *testing.T) { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprint(reverse), func(t *testing.T) { + f := newProjectionFixture(t) + devices := []string{"device-a", "device-b", "device-c"} + if reverse { + slices.Reverse(devices) + } + manifests := map[string]rawsync.CanonicalManifest{} + receipts := map[string]string{} + byAlias := map[string]string{} + for _, device := range devices { + m, r := f.accept(t, device, "initial-"+device, "") + out := projectionOutcome("shared pin") + out.Outcome.Results[0].Result.Messages[1].Content = device + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + manifests[device] = m + receipts[device] = r.Receipt + byAlias[f.alias(t, m)] = device + } + aliases := make([]string, 0, len(byAlias)) + for alias := range byAlias { + aliases = append(aliases, alias) + } + slices.Sort(aliases) + a, b, c := aliases[0], aliases[1], aliases[2] + for i, alias := range aliases { + require.NoError(t, f.sink.SetCuration(t.Context(), alias, "display_name", []string{"alpha", "beta", "gamma"}[i])) + require.NoError(t, f.sink.SetPin(t.Context(), alias, 0, true, []string{"one", "two", "three"}[i])) + } + require.NoError(t, f.sink.SetCuration(t.Context(), c, "starred", true)) + for _, alias := range []string{a, b} { + device := byAlias[alias] + m, r := f.accept(t, device, "equal-"+device, receipts[device]) + receipts[device] = r.Receipt + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("shared pin"))) + } + ar, err := f.sink.Resolve(t.Context(), a) + require.NoError(t, err) + br, err := f.sink.Resolve(t.Context(), b) + require.NoError(t, err) + assert.Equal(t, ar.SessionID, br.SessionID) + store := &Store{pg: f.runtime} + session, err := store.GetSession(t.Context(), ar.SessionID) + require.NoError(t, err) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "alpha", *session.DisplayName) + pins, err := store.ListPinnedMessages(t.Context(), ar.SessionID, "") + require.NoError(t, err) + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "one", *pins[0].Note) + require.NoError(t, f.sink.SetCuration(t.Context(), b, "display_name", "cohort")) + require.NoError(t, f.sink.SetPin(t.Context(), b, 0, false, "")) + cr, err := f.sink.Resolve(t.Context(), c) + require.NoError(t, err) + pins, err = store.ListPinnedMessages(t.Context(), cr.SessionID, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, "three", *pins[0].Note) + stars, err := store.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{cr.SessionID}, stars) + device := byAlias[a] + m, _ := f.accept(t, device, "split", receipts[device]) + out := projectionOutcome("shared pin") + out.Outcome.Results[0].Result.Messages[1].Content = "split" + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + for _, alias := range []string{a, b} { + r, err := f.sink.Resolve(t.Context(), alias) + require.NoError(t, err) + session, err := store.GetSession(t.Context(), r.SessionID) + require.NoError(t, err) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "cohort", *session.DisplayName) + pins, err := store.ListPinnedMessages(t.Context(), r.SessionID, "") + require.NoError(t, err) + assert.Empty(t, pins) + } + }) + } +} diff --git a/internal/postgres/raw_projection_concurrency_pgtest_test.go b/internal/postgres/raw_projection_concurrency_pgtest_test.go new file mode 100644 index 0000000000..6b78a0fdf8 --- /dev/null +++ b/internal/postgres/raw_projection_concurrency_pgtest_test.go @@ -0,0 +1,96 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/rawderive" + "testing" + "time" +) + +func TestRawProjectionCurationResolvesMembershipAfterConcurrentSplit(t *testing.T) { + f := newProjectionFixture(t) + a, ar := f.accept(t, "device-a", "race-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("equal"))) + b, _ := f.accept(t, "device-b", "race-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("equal"))) + aa := f.alias(t, a) + ba := f.alias(t, b) + resolved, err := f.sink.Resolve(t.Context(), aa) + require.NoError(t, err) + next, _ := f.accept(t, "device-a", "split-a", ar.Receipt) + lease := f.lease(t, next) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + gate, err := f.runtime.BeginTx(ctx, nil) + require.NoError(t, err) + defer gate.Rollback() + _, err = gate.ExecContext(ctx, `SELECT group_id FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, resolved.GroupID) + require.NoError(t, err) + projected := make(chan error, 1) + go func() { projected <- f.sink.Project(ctx, lease, next, projectionOutcome("split")) }() + waiting := func(count int) func() bool { + return func() bool { + var got int + err := f.admin.QueryRowContext(ctx, `SELECT count(*) FROM pg_stat_activity WHERE usename=$1 AND wait_event_type='Lock'`, f.role).Scan(&got) + return err == nil && got >= count + } + } + require.Eventually(t, waiting(1), 3*time.Second, 10*time.Millisecond) + curated := make(chan error, 1) + go func() { curated <- f.sink.SetCuration(ctx, aa, "starred", true) }() + require.Eventually(t, waiting(2), 3*time.Second, 10*time.Millisecond) + require.NoError(t, gate.Commit()) + require.NoError(t, <-projected) + require.NoError(t, <-curated) + ra, err := f.sink.Resolve(t.Context(), aa) + require.NoError(t, err) + rb, err := f.sink.Resolve(t.Context(), ba) + require.NoError(t, err) + assert.NotEqual(t, ra.SessionID, rb.SessionID) + stars, err := (&Store{pg: f.runtime}).ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{ra.SessionID}, stars) +} +func TestRawProjectionCurationSQLFailureRollsBackOverlayAndMaterialization(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "curation-failure", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("hello"))) + before, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + _, err = f.admin.ExecContext(t.Context(), `CREATE FUNCTION reject_star_materialization() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic star dependency unavailable'; END $$; CREATE TRIGGER reject_star_materialization BEFORE INSERT ON starred_sessions FOR EACH ROW EXECUTE FUNCTION reject_star_materialization()`) + require.NoError(t, err) + err = f.sink.SetCuration(t.Context(), "codex:portable", "starred", true) + require.Error(t, err) + assert.Contains(t, err.Error(), "synthetic star dependency unavailable") + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_curation`).Scan(&count)) + assert.Zero(t, count) + after, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, before.CorpusRevision, after.CorpusRevision) + stars, err := (&Store{pg: f.runtime}).ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Empty(t, stars) +} + +func TestRawProjectionRollsBackWhenLeaseExpiresDuringWrites(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "expires-during-write", "") + _, err := f.sink.SelectSourceGeneration(t.Context(), m, "parser-1") + require.NoError(t, err) + _, err = f.admin.ExecContext(t.Context(), `CREATE FUNCTION delay_projection_message() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.ordinal=0 THEN IF NOT EXISTS(SELECT 1 FROM raw_ingest_jobs WHERE state='leased' AND lease_expires_at>clock_timestamp()) THEN RAISE EXCEPTION 'fixture lease expired before writes'; END IF; PERFORM pg_sleep(0.4); END IF; RETURN NEW; END $$; CREATE TRIGGER delay_projection_message BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION delay_projection_message()`) + require.NoError(t, err) + leases, err := f.jobs.ClaimRawParseJobs(t.Context(), "expiry-worker", 1, 250*time.Millisecond) + require.NoError(t, err) + require.Len(t, leases, 1) + require.ErrorIs(t, f.sink.Project(t.Context(), leases[0], m, projectionOutcome("expired write")), rawderive.ErrLeaseLost) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Zero(t, count) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_source_contributions`).Scan(&count)) + assert.Zero(t, count) +} diff --git a/internal/postgres/raw_projection_content.go b/internal/postgres/raw_projection_content.go new file mode 100644 index 0000000000..c8c1578abd --- /dev/null +++ b/internal/postgres/raw_projection_content.go @@ -0,0 +1,203 @@ +package postgres + +import ( + "bytes" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/rawsync" +) + +func rawDigest(parts ...string) string { + h := sha256.New() + for _, p := range parts { + fmt.Fprintf(h, "%d:", len(p)) + h.Write([]byte(p)) + } + return hex.EncodeToString(h.Sum(nil)) +} +func rawSourceID(m rawsync.CanonicalManifest) string { + return rawDigest("source-v1", m.Identity.TenantID, m.Identity.DeviceID, string(m.Manifest.Provider), m.Manifest.ConfiguredRootID, m.Manifest.SourceKey) +} +func rawGroupID(m rawsync.CanonicalManifest, s db.Session) (string, string) { + key := s.SourceSessionID + if key == "" { + key = rawDigest("source-local-v1", rawSourceID(m), s.ID) + } + return rawDigest("group-v1", m.Identity.TenantID, s.Agent, key), key +} +func encodeRawPayload(p ingest.PreparedSession) ([]byte, error) { + var b bytes.Buffer + err := gob.NewEncoder(&b).Encode(p) + return b.Bytes(), err +} +func decodeRawPayload(b []byte) (ingest.PreparedSession, error) { + var p ingest.PreparedSession + err := gob.NewDecoder(bytes.NewReader(b)).Decode(&p) + return p, err +} + +// rawContentRevision hashes a versioned structural representation of every +// normalized field, including fields omitted from public JSON. Only the explicit +// non-content fields below are removed. JSON-valued +// fields are canonical structures, not their formatting or object key order. +func rawContentRevision(p ingest.PreparedSession) (string, error) { + s := &p.Session + s.ID = "" + s.Machine = "" + s.DisplayName = nil + s.CreatedAt = "" + s.LocalModifiedAt = nil + s.TranscriptRevision = nil + s.DeletedAt = nil + s.DeletionCause = nil + s.SourceMissingAt = nil + s.FilePath = nil + s.FileSize = nil + s.FileMtime = nil + s.FileInode = nil + s.FileDevice = nil + s.FileHash = nil + s.NextOrdinal = 0 + s.LastEntryUUID = nil + s.ClaudeLinearParse = nil + s.LastWriteIncremental = false + s.PreserveSessionName = false + s.PreserveStoredAutomation = false + s.DataVersion = 0 + s.SecretsRulesVersion = "" + s.QualitySignalVersion = 0 + // Recency-derived state is published separately from immutable content. + s.SignalsPendingSince, s.HealthScore, s.HealthGrade = nil, nil, nil + s.Outcome, s.OutcomeConfidence = "", "" + if s.QualitySignals != nil { + copy := *s.QualitySignals + copy.Version = 0 + s.QualitySignals = © + } + p.Signals.FullState = nil + p.Signals.SecretsRulesVersion = "" + p.Signals.QualitySignals.Version = 0 + p.Signals.SignalsPendingSince, p.Signals.HealthScore, p.Signals.HealthGrade = nil, nil, nil + p.Signals.Outcome, p.Signals.OutcomeConfidence = "", "" + p.Validation = db.ValidationStats{} + // Copy nested slices before removing transport IDs: callers retain their rows. + var err error + p.Messages = append([]db.Message(nil), p.Messages...) + for i := range p.Messages { + m := &p.Messages[i] + m.ID = 0 + m.SessionID = "" + m.ToolResults = nil + m.ToolCalls = append([]db.ToolCall(nil), m.ToolCalls...) + for j := range m.ToolCalls { + tc := &m.ToolCalls[j] + tc.MessageID = 0 + tc.Rendering = "" + tc.SessionID = "" + tc.ResultEvents = append([]db.ToolResultEvent(nil), tc.ResultEvents...) + for k := range tc.ResultEvents { + tc.ResultEvents[k].RawContentDigest = nil + tc.ResultEvents[k].SummaryParticipates = nil + } + } + } + p.UsageEvents = append([]db.UsageEvent(nil), p.UsageEvents...) + for i := range p.UsageEvents { + p.UsageEvents[i].ID = 0 + p.UsageEvents[i].SessionID = "" + } + p.Findings = append([]db.SecretFinding(nil), p.Findings...) + for i := range p.Findings { + p.Findings[i].SessionID = "" + p.Findings[i].RulesVersion = "" + } + structure, err := rawCanonicalValue(reflect.ValueOf(p), "") + if err != nil { + return "", err + } + encoded, err := json.Marshal(structure) + if err != nil { + return "", err + } + return rawDigest("normalized-content-v1", string(encoded)), nil +} +func rawCanonicalValue(v reflect.Value, field string) (any, error) { + if !v.IsValid() { + return nil, nil + } + if v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return nil, nil + } + return rawCanonicalValue(v.Elem(), field) + } + if field == "TokenUsage" || field == "InputJSON" { + var b []byte + if v.Kind() == reflect.String { + b = []byte(v.String()) + } else if v.Kind() == reflect.Slice { + b = v.Bytes() + } + if len(b) > 0 { + var value any + d := json.NewDecoder(bytes.NewReader(b)) + d.UseNumber() + if err := d.Decode(&value); err == nil { + return value, nil + } + } + } + switch v.Kind() { + case reflect.Struct: + out := map[string]any{} + for i := 0; i < v.NumField(); i++ { + f := v.Type().Field(i) + if !f.IsExported() { + continue + } + value, err := rawCanonicalValue(v.Field(i), f.Name) + if err != nil { + return nil, err + } + out[f.Name] = value + } + return out, nil + case reflect.Slice, reflect.Array: + out := make([]any, v.Len()) + for i := range out { + value, err := rawCanonicalValue(v.Index(i), "") + if err != nil { + return nil, err + } + out[i] = value + } + return out, nil + case reflect.Map: + out := map[string]any{} + iter := v.MapRange() + for iter.Next() { + value, err := rawCanonicalValue(iter.Value(), "") + if err != nil { + return nil, err + } + out[fmt.Sprint(iter.Key().Interface())] = value + } + return out, nil + default: + return v.Interface(), nil + } +} + +func rawMessageKey(m db.Message) (string, error) { + // Include ordinal and complete message content. A UUID or ordinal reused for + // different content must leave an old pin unresolved. + p := ingest.PreparedSession{Messages: []db.Message{m}} + return rawContentRevision(p) +} diff --git a/internal/postgres/raw_projection_content_test.go b/internal/postgres/raw_projection_content_test.go new file mode 100644 index 0000000000..1df612ceec --- /dev/null +++ b/internal/postgres/raw_projection_content_test.go @@ -0,0 +1,94 @@ +package postgres + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" +) + +func TestRawContentRevisionRetainsHiddenSemanticFields(t *testing.T) { + original := ingest.PreparedSession{Session: db.Session{ID: "physical", Machine: "device", SessionName: new("provider title")}, Messages: []db.Message{{Ordinal: 0, Role: "assistant", Content: "same", ToolCalls: []db.ToolCall{{ToolName: "Bash", InputJSON: `{"a":1,"b":2}`, ResultEvents: []db.ToolResultEvent{{Content: "same", RawContentDigest: []byte("transport")}}}}}}} + digest, err := rawContentRevision(original) + require.NoError(t, err) + clone := func() ingest.PreparedSession { + b, err := encodeRawPayload(original) + require.NoError(t, err) + p, err := decodeRawPayload(b) + require.NoError(t, err) + return p + } + t.Run("transport", func(t *testing.T) { + p := clone() + p.Session.ID = "other" + p.Session.Machine = "other" + p.Session.FilePath = new("capture.jsonl") + p.Session.FileMtime = new(int64(99)) + p.Messages[0].ToolCalls[0].ResultEvents[0].RawContentDigest = []byte("other transport") + p.Messages[0].ToolCalls[0].InputJSON = `{ "b": 2, "a": 1 }` + got, err := rawContentRevision(p) + require.NoError(t, err) + assert.Equal(t, digest, got) + }) + t.Run("derivation version metadata", func(t *testing.T) { + p := clone() + p.Session.DataVersion = 123 + p.Signals.SecretsRulesVersion = "next-rules" + p.Signals.QualitySignals.Version = 123 + got, err := rawContentRevision(p) + require.NoError(t, err) + assert.Equal(t, digest, got) + }) + t.Run("hidden provider title", func(t *testing.T) { + p := clone() + p.Session.SessionName = new("changed") + got, err := rawContentRevision(p) + require.NoError(t, err) + assert.NotEqual(t, digest, got) + }) + t.Run("explicit zero usage presence", func(t *testing.T) { + p := clone() + p.Messages[0].HasOutputTokens = true + got, err := rawContentRevision(p) + require.NoError(t, err) + assert.NotEqual(t, digest, got) + }) + t.Run("semantic tool path", func(t *testing.T) { + p := clone() + p.Messages[0].ToolCalls[0].FilePath = "src/changed.go" + got, err := rawContentRevision(p) + require.NoError(t, err) + assert.NotEqual(t, digest, got) + }) +} + +func TestRawContentRevisionIgnoresOnlyRecencyDerivedState(t *testing.T) { + p := ingest.PreparedSession{Session: db.Session{EndedAt: new("2026-09-11T12:00:00Z"), MessageCount: 3}, Messages: []db.Message{{Ordinal: 0, Role: "user", Content: "task"}, {Ordinal: 1, Role: "assistant", Content: "working"}, {Ordinal: 2, Role: "user", Content: "continue"}}} + p.Signals.Outcome, p.Signals.OutcomeConfidence = "unknown", "low" + p.Signals.SignalsPendingSince = new("2026-09-11T12:01:00Z") + copyRawSignalFields(&p.Session, p.Signals) + first, err := rawContentRevision(p) + require.NoError(t, err) + p.Signals.SignalsPendingSince = new("2026-09-11T12:02:00Z") + copyRawSignalFields(&p.Session, p.Signals) + second, err := rawContentRevision(p) + require.NoError(t, err) + assert.Equal(t, first, second, "equal content at a second recent instant") + p.Signals.SignalsPendingSince = nil + p.Signals.Outcome, p.Signals.OutcomeConfidence = "abandoned", "medium" + p.Signals.HealthScore, p.Signals.HealthGrade = new(85), new("B") + copyRawSignalFields(&p.Session, p.Signals) + settled, err := rawContentRevision(p) + require.NoError(t, err) + assert.Equal(t, first, settled, "recency expiry is not new content") + p.Signals.ToolRetryCount++ + stableSignal, err := rawContentRevision(p) + require.NoError(t, err) + assert.NotEqual(t, settled, stableSignal, "stable derived signals remain content") + p.Messages[2].Content = "different task" + semantic, err := rawContentRevision(p) + require.NoError(t, err) + assert.NotEqual(t, stableSignal, semantic, "actual transcript semantics remain content") +} diff --git a/internal/postgres/raw_projection_curation.go b/internal/postgres/raw_projection_curation.go new file mode 100644 index 0000000000..88b7647109 --- /dev/null +++ b/internal/postgres/raw_projection_curation.go @@ -0,0 +1,211 @@ +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" +) + +// RawIdentityError lets the hosted boundary distinguish conflict from gone +// without accepting a private row ID or selecting an arbitrary cohort. +type RawIdentityError struct{ Identity RawIdentity } + +func (e *RawIdentityError) Error() string { + return "raw session identity is " + string(e.Identity.State) +} + +func (s *RawProjectionStore) lockRawCurationTarget(ctx context.Context, tx *sql.Tx, alias string) (RawIdentity, error) { + rows, err := tx.QueryContext(ctx, `SELECT g.group_id FROM raw_session_groups g JOIN raw_session_public_aliases a ON a.group_id=g.group_id WHERE a.alias_id=$1 ORDER BY g.group_id FOR UPDATE OF g`, alias) + if err != nil { + return RawIdentity{}, err + } + for rows.Next() { + var group string + if err = rows.Scan(&group); err != nil { + rows.Close() + return RawIdentity{}, err + } + } + err = rows.Err() + rows.Close() + if err != nil { + return RawIdentity{}, err + } + resolved, err := resolveRawIdentity(ctx, tx, alias) + if err != nil { + return resolved, err + } + if resolved.State != RawIdentityUnique { + return resolved, &RawIdentityError{resolved} + } + if s.curationGuard != nil { + if err := s.curationGuard(ctx, tx, alias, resolved); err != nil { + return resolved, err + } + } + return resolved, nil +} + +// SetCuration changes exactly one field on the current target. A base alias +// supersedes the same field's branch overrides, including inactive branches. +// A variant action writes every active member of its current content cohort. +func (s *RawProjectionStore) SetCuration(ctx context.Context, alias, field string, value any) error { + switch field { + case "starred", "trashed": + if _, ok := value.(bool); !ok { + return fmt.Errorf("curation field requires boolean") + } + case "display_name": + if value != nil { + if _, ok := value.(string); !ok { + return fmt.Errorf("display name requires string or null") + } + } + default: + return fmt.Errorf("unsupported raw curation field") + } + encoded, err := json.Marshal(value) + if err != nil { + return err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + target, err := s.lockRawCurationTarget(ctx, tx, alias) + if err != nil { + return err + } + if target.AnchorBranch == "" { + _, err = tx.ExecContext(ctx, `DELETE FROM raw_curation WHERE group_id=$1 AND field=$2 AND branch_id<>''`, target.GroupID, field) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_curation(group_id,branch_id,field,value) VALUES($1,'',$2,$3) ON CONFLICT(group_id,branch_id,field) DO UPDATE SET value=EXCLUDED.value`, target.GroupID, field, string(encoded)) + } else { + members, memberErr := rawCurationMembers(ctx, tx, target) + if memberErr != nil { + return memberErr + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_curation(group_id,branch_id,field,value) SELECT group_id,branch_id,$3,$4::jsonb FROM raw_session_branches WHERE group_id=$1 AND session_id=$2 AND active AND branch_id=ANY($5) ON CONFLICT(group_id,branch_id,field) DO UPDATE SET value=EXCLUDED.value`, target.GroupID, target.SessionID, field, string(encoded), members) + } + if err != nil { + return err + } + if err = s.publishRawCuration(ctx, tx, target.GroupID); err != nil { + return err + } + return tx.Commit() +} +func (s *RawProjectionStore) publishRawCuration(ctx context.Context, tx *sql.Tx, group string) error { + var revision int64 + err := tx.QueryRowContext(ctx, `UPDATE raw_corpus_state SET corpus_revision=corpus_revision+1 WHERE singleton=1 RETURNING corpus_revision`).Scan(&revision) + if err != nil { + return err + } + return s.materializeGroup(ctx, tx, group, revision) +} + +type rawOverlays map[string]map[string]json.RawMessage + +func loadRawOverlays(ctx context.Context, tx *sql.Tx, group string) (rawOverlays, error) { + rows, err := tx.QueryContext(ctx, `SELECT branch_id,field,value FROM raw_curation WHERE group_id=$1`, group) + if err != nil { + return nil, err + } + defer rows.Close() + out := rawOverlays{} + for rows.Next() { + var branch, field string + var value []byte + if err = rows.Scan(&branch, &field, &value); err != nil { + return nil, err + } + if out[branch] == nil { + out[branch] = map[string]json.RawMessage{} + } + out[branch][field] = value + } + return out, rows.Err() +} +func (o rawOverlays) effective(branch, field string) json.RawMessage { + if value, ok := o[branch][field]; ok { + return value + } + return o[""][field] +} +func (o rawOverlays) boolean(branch, field string) bool { + var value bool + _ = json.Unmarshal(o.effective(branch, field), &value) + return value +} +func (s *RawProjectionStore) materializeCuration(ctx context.Context, tx *sql.Tx, group, id string, members []rawBranch) error { + overlays, err := loadRawOverlays(ctx, tx, group) + if err != nil { + return err + } + visible := make([]rawBranch, 0, len(members)) + for _, b := range members { + if !overlays.boolean(b.ID, "trashed") { + visible = append(visible, b) + } + } + trashed := len(visible) == 0 + if trashed { + visible = members + } + starred := false + var name *string + selectedName := false + for _, b := range visible { + if !trashed && overlays.boolean(b.ID, "starred") { + starred = true + } + if raw, present := overlays[b.ID]["display_name"]; present && !selectedName { + if err = json.Unmarshal(raw, &name); err != nil { + return err + } + selectedName = true + } + } + if !selectedName { + if raw, ok := overlays[""]["display_name"]; ok { + if err = json.Unmarshal(raw, &name); err != nil { + return err + } + } + } + _, err = tx.ExecContext(ctx, `UPDATE sessions SET display_name=$2,deleted_at=CASE WHEN $3 THEN COALESCE(deleted_at,clock_timestamp()) ELSE NULL END,deletion_cause=CASE WHEN $3 THEN 'user' ELSE NULL END WHERE id=$1`, id, name, trashed) + if err != nil { + return err + } + if starred { + _, err = tx.ExecContext(ctx, `INSERT INTO starred_sessions(session_id) VALUES($1) ON CONFLICT(session_id) DO NOTHING`, id) + } else { + _, err = tx.ExecContext(ctx, `DELETE FROM starred_sessions WHERE session_id=$1`, id) + } + if err != nil { + return err + } + return materializeRawPins(ctx, tx, group, id, visible) +} + +func rawCurationMembers(ctx context.Context, tx *sql.Tx, target RawIdentity) ([]string, error) { + branches, err := loadRawBranches(ctx, tx, "group_id", target.GroupID) + if err != nil { + return nil, err + } + overlays, err := loadRawOverlays(ctx, tx, target.GroupID) + if err != nil { + return nil, err + } + var members []string + for _, b := range branches { + if b.Active && b.Session == target.SessionID && !overlays.boolean(b.ID, "excluded") { + members = append(members, b.ID) + } + } + return members, nil +} diff --git a/internal/postgres/raw_projection_exclusions.go b/internal/postgres/raw_projection_exclusions.go new file mode 100644 index 0000000000..4734ba0557 --- /dev/null +++ b/internal/postgres/raw_projection_exclusions.go @@ -0,0 +1,120 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" +) + +// ExcludeTrashedSession retains accepted custody and source proof, but durably +// excludes the resolved group/cohort from every physical read inventory. +func (s *RawProjectionStore) ExcludeTrashedSession(ctx context.Context, alias string) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + target, err := s.lockRawCurationTarget(ctx, tx, alias) + if err != nil { + var identityErr *RawIdentityError + if errors.As(err, &identityErr) && identityErr.Identity.State == RawIdentityGone { + return false, nil + } + return false, err + } + var trashed bool + err = tx.QueryRowContext(ctx, `SELECT deleted_at IS NOT NULL FROM sessions WHERE id=$1`, target.SessionID).Scan(&trashed) + if err != nil { + return false, err + } + if !trashed { + return false, nil + } + if target.AnchorBranch == "" { + _, err = tx.ExecContext(ctx, `DELETE FROM raw_curation WHERE group_id=$1 AND field='excluded' AND branch_id<>''`, target.GroupID) + if err != nil { + return false, err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_curation(group_id,branch_id,field,value) VALUES($1,'','excluded','true') ON CONFLICT(group_id,branch_id,field) DO UPDATE SET value='true'`, target.GroupID) + } else { + err = excludeRawCohort(ctx, tx, target.GroupID, target.SessionID) + } + if err != nil { + return false, err + } + if err = s.publishRawExclusion(ctx, tx, target.GroupID); err != nil { + return false, err + } + return true, tx.Commit() +} + +// EmptyTrash selects only fully trashed current cohorts under sorted group +// locks. An invisible branch in an otherwise visible cohort is not selected. +func (s *RawProjectionStore) EmptyTrash(ctx context.Context) (int, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + rows, err := tx.QueryContext(ctx, `SELECT g.group_id FROM raw_session_groups g WHERE EXISTS(SELECT 1 FROM sessions s WHERE s.raw_group_id=g.group_id AND s.deleted_at IS NOT NULL) ORDER BY g.group_id FOR UPDATE OF g`) + if err != nil { + return 0, err + } + var groups []string + for rows.Next() { + var group string + if err = rows.Scan(&group); err != nil { + rows.Close() + return 0, err + } + groups = append(groups, group) + } + err = rows.Err() + rows.Close() + if err != nil { + return 0, err + } + count := 0 + for _, group := range groups { + rows, err := tx.QueryContext(ctx, `SELECT id FROM sessions WHERE raw_group_id=$1 AND raw_group_id<>'' AND deleted_at IS NOT NULL ORDER BY id`, group) + if err != nil { + return 0, err + } + var ids []string + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + rows.Close() + return 0, err + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return 0, err + } + for _, id := range ids { + if err = excludeRawCohort(ctx, tx, group, id); err != nil { + return 0, err + } + count++ + } + if len(ids) > 0 { + if err = s.publishRawExclusion(ctx, tx, group); err != nil { + return 0, err + } + } + } + return count, tx.Commit() +} +func excludeRawCohort(ctx context.Context, tx *sql.Tx, group, id string) error { + _, err := tx.ExecContext(ctx, `INSERT INTO raw_curation(group_id,branch_id,field,value) SELECT group_id,branch_id,'excluded','true'::jsonb FROM raw_session_branches WHERE group_id=$1 AND session_id=$2 AND active ON CONFLICT(group_id,branch_id,field) DO UPDATE SET value='true'`, group, id) + return err +} +func (s *RawProjectionStore) publishRawExclusion(ctx context.Context, tx *sql.Tx, group string) error { + if _, err := tx.ExecContext(ctx, `UPDATE raw_corpus_state SET identity_revision=identity_revision+1 WHERE singleton=1`); err != nil { + return err + } + return s.publishRawCuration(ctx, tx, group) +} diff --git a/internal/postgres/raw_projection_exclusions_pgtest_test.go b/internal/postgres/raw_projection_exclusions_pgtest_test.go new file mode 100644 index 0000000000..ef2e4b2278 --- /dev/null +++ b/internal/postgres/raw_projection_exclusions_pgtest_test.go @@ -0,0 +1,67 @@ +//go:build pgtest + +package postgres + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestRawProjectionEmptyTrashExcludesOnlyFullyTrashedCohorts(t *testing.T) { + f := newProjectionFixture(t) + a, ar := f.accept(t, "device-a", "exclude-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("equal"))) + aa := f.alias(t, a) + require.NoError(t, f.sink.SetCuration(t.Context(), aa, "display_name", "retained excluded name")) + require.NoError(t, f.sink.SetCuration(t.Context(), aa, "trashed", true)) + b, _ := f.accept(t, "device-b", "exclude-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("equal"))) + removed, err := f.sink.EmptyTrash(t.Context()) + require.NoError(t, err) + assert.Zero(t, removed) + next, nr := f.accept(t, "device-a", "split-excluded", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("split"))) + removed, err = f.sink.EmptyTrash(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, removed) + r, err := f.sink.Resolve(t.Context(), aa) + require.NoError(t, err) + assert.Equal(t, RawIdentityGone, r.State) + newest, _ := f.accept(t, "device-a", "excluded-reparse", nr.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, newest), newest, projectionOutcome("equal"))) + + require.NoError(t, f.sink.SetCuration(t.Context(), f.alias(t, b), "display_name", "visible name")) + var name string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT c.value #>> '{}' FROM raw_curation c JOIN raw_session_public_aliases a ON a.anchor_branch=c.branch_id WHERE a.alias_id=$1 AND c.field='display_name'`, aa).Scan(&name)) + assert.Equal(t, "retained excluded name", name) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Equal(t, 1, count) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM session_sources`).Scan(&count)) + assert.Equal(t, 2, count, "user exclusion retains source proof") + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_session_branches WHERE active`).Scan(&count)) + assert.Equal(t, 2, count) + trash, err := (&Store{pg: f.runtime}).ListTrashedSessions(t.Context()) + require.NoError(t, err) + assert.Empty(t, trash) +} +func TestRawProjectionBaseExclusionAlsoCoversFutureBranches(t *testing.T) { + f := newProjectionFixture(t) + a, _ := f.accept(t, "device-a", "base-exclude-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("content"))) + removed, err := f.sink.ExcludeTrashedSession(t.Context(), "codex:portable") + require.NoError(t, err) + assert.False(t, removed) + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "trashed", true)) + removed, err = f.sink.ExcludeTrashedSession(t.Context(), "codex:portable") + require.NoError(t, err) + assert.True(t, removed) + b, _ := f.accept(t, "device-b", "base-exclude-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("new source"))) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Zero(t, count) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM session_sources`).Scan(&count)) + assert.Equal(t, 2, count) +} diff --git a/internal/postgres/raw_projection_fence.go b/internal/postgres/raw_projection_fence.go new file mode 100644 index 0000000000..389dd429bc --- /dev/null +++ b/internal/postgres/raw_projection_fence.go @@ -0,0 +1,199 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +type RawProjectionOptions struct { + Tenant string + Content ingest.ContentOptions + RetryPolicy rawderive.RetryPolicy + // Now supplies the observation clock for derived signal settling. + Now func() time.Time +} +type RawProjectionStore struct { + curationGuard func(context.Context, *sql.Tx, string, RawIdentity) error + db *sql.DB + options RawProjectionOptions +} + +var _ rawderive.ProjectionSink = (*RawProjectionStore)(nil) + +func NewRawProjectionStore(database *sql.DB, options RawProjectionOptions) (*RawProjectionStore, error) { + if options.Now == nil { + options.Now = time.Now + } + if options.RetryPolicy == (rawderive.RetryPolicy{}) { + options.RetryPolicy = rawderive.RetryPolicy{Base: time.Second, Maximum: time.Minute, MaxAttempts: 5} + } + if err := options.RetryPolicy.Validate(); err != nil { + return nil, err + } + if database == nil { + return nil, fmt.Errorf("raw projection requires a database") + } + var schema string + if err := database.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + return nil, err + } + if err := CheckHostedTenant(context.Background(), database, schema, options.Tenant); err != nil { + return nil, err + } + if err := CheckSchemaCompat(context.Background(), database); err != nil { + return nil, fmt.Errorf("hosted schema requires owner provisioning: %w", err) + } + rows, err := database.Query(`SELECT s.provenance_kind,s.raw_group_id,s.raw_content_revision,j.projection_generation,j.projection_selected,c.recency_state FROM sessions s CROSS JOIN raw_ingest_jobs j CROSS JOIN raw_content_revisions c LIMIT 0`) + if err != nil { + return nil, fmt.Errorf("raw projection schema requires owner provisioning: %w", err) + } + if err = rows.Close(); err != nil { + return nil, err + } + var indexes bool + if err := database.QueryRow(`SELECT to_regclass('raw_selected_job') IS NOT NULL AND to_regclass('raw_pending_signals') IS NOT NULL AND to_regclass('raw_hosted_jobs_due') IS NOT NULL AND to_regclass('raw_hosted_jobs_expired') IS NOT NULL`).Scan(&indexes); err != nil { + return nil, err + } + if !indexes { + return nil, fmt.Errorf("raw runtime indexes require owner provisioning") + } + return &RawProjectionStore{db: database, options: options}, nil +} + +// SelectSourceGeneration is an explicit scheduler decision made before claiming +// a job. Equal selections are idempotent, including after successful completion. +// Version strings have no ordering; projection never invokes this operation. +func (s *RawProjectionStore) SelectSourceGeneration(ctx context.Context, m rawsync.CanonicalManifest, version string) (int64, error) { + if err := s.validateManifest(m); err != nil { + return 0, err + } + if err := validateRawIngestProcessingVersion(version); err != nil { + return 0, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + if err = s.lockHead(ctx, tx, m); err != nil { + return 0, err + } + generation, err := selectRawSourceGenerationTx(ctx, tx, m, version) + if err != nil { + return 0, err + } + return generation, tx.Commit() +} + +// selectRawSourceGenerationTx requires the caller to own the current source-head +// lock and to have verified the canonical manifest. Acceptance uses the same +// transaction as its receipt; explicit rollout uses the public wrapper above. +func selectRawSourceGenerationTx(ctx context.Context, tx *sql.Tx, m rawsync.CanonicalManifest, version string) (int64, error) { + var err error + source := rawSourceID(m) + var generation, previousJob int64 + var manifest, selected string + err = tx.QueryRowContext(ctx, `SELECT projection_generation,selected_manifest_id,processing_version,selected_job_id FROM raw_source_projections WHERE tenant_id=$1 AND source_id=$2`, m.Identity.TenantID, source).Scan(&generation, &manifest, &selected, &previousJob) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + if manifest == m.ManifestID && selected == version { + return generation, nil + } + generation++ + var jobID int64 + err = tx.QueryRowContext(ctx, `INSERT INTO raw_ingest_jobs(tenant_id,manifest_id,stage,processing_version,projection_generation) VALUES($1,$2,'parse',$3,$4) + ON CONFLICT(tenant_id,manifest_id,stage,processing_version) DO UPDATE SET projection_generation=EXCLUDED.projection_generation,projection_selected=true,attempt_count=0,state='ready',available_at=clock_timestamp(),lease_owner='',lease_expires_at=NULL RETURNING id`, m.Identity.TenantID, m.ManifestID, version, generation).Scan(&jobID) + if err != nil { + return 0, err + } + + if previousJob != 0 && previousJob != jobID { + _, err = tx.ExecContext(ctx, `UPDATE raw_ingest_jobs SET projection_selected=false,state=CASE WHEN state IN ('leased','ready','retrying') THEN 'superseded' ELSE state END,lease_owner='',lease_expires_at=NULL WHERE tenant_id=$1 AND id=$2`, m.Identity.TenantID, previousJob) + if err != nil { + return 0, err + } + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_source_projections(source_id,device_id,provider,configured_root_id,source_key_sha256,selected_manifest_id,processing_version,projection_generation,selected_job_id) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT(source_id) DO UPDATE SET selected_manifest_id=EXCLUDED.selected_manifest_id,processing_version=EXCLUDED.processing_version,projection_generation=EXCLUDED.projection_generation,selected_job_id=EXCLUDED.selected_job_id`, source, m.Identity.DeviceID, string(m.Manifest.Provider), m.Manifest.ConfiguredRootID, rawIngestKeyDigest(m.Manifest.SourceKey), m.ManifestID, version, generation, jobID) + if err != nil { + return 0, err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_projection_generations(source_id,generation,manifest_id,processing_version) VALUES($1,$2,$3,$4)`, source, generation, m.ManifestID, version) + if err != nil { + return 0, err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_corpus_state(singleton,selection_revision) VALUES(1,1) ON CONFLICT(singleton) DO UPDATE SET selection_revision=raw_corpus_state.selection_revision+1`) + if err != nil { + return 0, err + } + return generation, nil +} +func (s *RawProjectionStore) validateManifest(m rawsync.CanonicalManifest) error { + if m.Identity.TenantID != s.options.Tenant { + return rawderive.ErrLeaseLost + } + return validateRawIngestCanonicalManifest(m) +} +func (s *RawProjectionStore) lockHead(ctx context.Context, tx *sql.Tx, m rawsync.CanonicalManifest) error { + head, err := lockRawIngestHead(ctx, tx, m) + if errors.Is(err, sql.ErrNoRows) { + return rawderive.ErrLeaseLost + } + if err != nil { + return err + } + if head.ManifestID != m.ManifestID { + return rawderive.ErrLeaseLost + } + var exact bool + err = tx.QueryRowContext(ctx, `SELECT canonical_json=$3 FROM raw_manifests WHERE tenant_id=$1 AND manifest_id=$2`, s.options.Tenant, m.ManifestID, m.CanonicalJSON).Scan(&exact) + if err != nil { + return err + } + if !exact { + return rawderive.ErrLeaseLost + } + return nil +} +func (s *RawProjectionStore) lockLease(ctx context.Context, tx *sql.Tx, m rawsync.CanonicalManifest, l rawderive.JobLease) error { + if l.Identity != m.Identity || l.ManifestID != m.ManifestID || l.ProjectionGeneration <= 0 { + return rawderive.ErrLeaseLost + } + var valid bool + err := tx.QueryRowContext(ctx, `SELECT job.state='leased' AND job.lease_owner=$3 AND job.attempt_count=$4 AND job.lease_expires_at>clock_timestamp() AND job.manifest_id=$5 AND job.processing_version=$6 AND job.projection_generation=$7 AND selected.selected_job_id=job.id AND selected.projection_generation=job.projection_generation AND selected.selected_manifest_id=job.manifest_id AND selected.processing_version=job.processing_version + FROM raw_ingest_jobs job JOIN raw_source_projections selected ON selected.tenant_id=job.tenant_id AND selected.source_id=$8 WHERE job.tenant_id=$1 AND job.id=$2 AND job.stage='parse' FOR UPDATE OF job`, s.options.Tenant, l.ID, l.Owner, l.Attempt, l.ManifestID, l.ProcessingVersion, l.ProjectionGeneration, rawSourceID(m)).Scan(&valid) + if errors.Is(err, sql.ErrNoRows) || err == nil && !valid { + return rawderive.ErrLeaseLost + } + return err +} +func completeProjectionJob(ctx context.Context, tx *sql.Tx, l rawderive.JobLease, complete bool, policy rawderive.RetryPolicy) (rawderive.CommittedProjectionOutcome, error) { + state := "complete" + var outcome rawderive.CommittedProjectionOutcome + var delay time.Duration + diagnostic := "" + if !complete { + decision := policy.Decide(l.Attempt) + delay = decision.Delay + diagnostic = "projection:incomplete" + state = "retrying" + outcome = rawderive.ProjectionRetrying + if decision.Failed { + state = "failed" + outcome = rawderive.ProjectionFailed + } + } + result, err := tx.ExecContext(ctx, `UPDATE raw_ingest_jobs SET state=$8,lease_owner='',lease_expires_at=NULL,available_at=clock_timestamp()+($9 * interval '1 microsecond'),last_error_class=CASE WHEN $10='' THEN '' ELSE 'projection' END,last_error=$10,updated_at=clock_timestamp() WHERE tenant_id=$1 AND id=$2 AND manifest_id=$3 AND processing_version=$4 AND projection_generation=$5 AND lease_owner=$6 AND attempt_count=$7 AND state='leased' AND lease_expires_at>clock_timestamp()`, l.Identity.TenantID, l.ID, l.ManifestID, l.ProcessingVersion, l.ProjectionGeneration, l.Owner, l.Attempt, state, delay.Microseconds(), diagnostic) + if err != nil { + return 0, err + } + return outcome, requireRawParseLeaseUpdate(result) +} diff --git a/internal/postgres/raw_projection_identity.go b/internal/postgres/raw_projection_identity.go new file mode 100644 index 0000000000..909d93a715 --- /dev/null +++ b/internal/postgres/raw_projection_identity.go @@ -0,0 +1,177 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "slices" +) + +type RawIdentityState string + +const ( + RawIdentityUnique RawIdentityState = "unique" + RawIdentityAmbiguous RawIdentityState = "ambiguous" + RawIdentityGone RawIdentityState = "gone" +) + +// RawIdentity resolves aliases only. SessionID is private storage metadata for +// the hosted adapter; callers must map it before returning a public response. +type RawIdentity struct { + State RawIdentityState + GroupID, AnchorBranch, SessionID, ContentRevision, PublicID string + Variants []string + ProjectionGeneration, SelectedGeneration, CorpusRevision, IdentityRevision, SelectionRevision int64 +} + +func (s *RawProjectionStore) Resolve(ctx context.Context, alias string) (RawIdentity, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) + if err != nil { + return RawIdentity{}, err + } + defer func() { _ = tx.Rollback() }() + result, err := resolveRawIdentity(ctx, tx, alias) + if err != nil { + return result, err + } + return result, tx.Commit() +} +func resolveRawIdentity(ctx context.Context, tx *sql.Tx, alias string) (RawIdentity, error) { + result := RawIdentity{State: RawIdentityGone} + stateErr := tx.QueryRowContext(ctx, `SELECT corpus_revision,identity_revision,selection_revision FROM raw_corpus_state WHERE singleton=1`).Scan(&result.CorpusRevision, &result.IdentityRevision, &result.SelectionRevision) + if stateErr != nil && !errors.Is(stateErr, sql.ErrNoRows) { + return result, stateErr + } + rows, err := tx.QueryContext(ctx, `SELECT group_id,anchor_branch FROM raw_session_public_aliases WHERE alias_id=$1 ORDER BY group_id`, alias) + if err != nil { + return result, err + } + type target struct{ group, anchor string } + var targets []target + for rows.Next() { + var t target + if err = rows.Scan(&t.group, &t.anchor); err != nil { + rows.Close() + return result, err + } + targets = append(targets, t) + } + err = rows.Err() + rows.Close() + if err != nil { + return result, err + } + if len(targets) == 0 { + return result, nil + } + if len(targets) > 1 { + var active []target + for _, t := range targets { + variants, err := rawGroupVariants(ctx, tx, t.group) + if err != nil { + return result, err + } + if len(variants) > 0 { + active = append(active, t) + result.Variants = append(result.Variants, variants...) + } + } + if len(active) > 1 { + result.State = RawIdentityAmbiguous + slices.Sort(result.Variants) + return result, nil + } + if len(active) == 0 { + return result, nil + } + targets = active + result.Variants = nil + } + + result.GroupID = targets[0].group + result.AnchorBranch = targets[0].anchor + branches, err := loadRawBranches(ctx, tx, "group_id", result.GroupID) + if err != nil { + return result, err + } + var base string + if err = tx.QueryRowContext(ctx, `SELECT base_alias FROM raw_session_groups WHERE group_id=$1`, result.GroupID).Scan(&base); err != nil { + return result, err + } + overlays, err := loadRawOverlays(ctx, tx, result.GroupID) + if err != nil { + return result, err + } + cohorts := map[string]rawBranch{} + var anchor *rawBranch + for _, b := range branches { + if !b.Active || overlays.boolean(b.ID, "excluded") { + continue + } + if _, ok := cohorts[b.Session]; !ok { + cohorts[b.Session] = b + } + if b.ID == result.AnchorBranch { + copy := b + anchor = © + } + } + for _, b := range cohorts { + result.Variants = append(result.Variants, base+"~"+b.ID) + } + slices.Sort(result.Variants) + if result.AnchorBranch != "" && anchor == nil || len(cohorts) == 0 { + return result, nil + } + if result.AnchorBranch == "" { + if len(cohorts) > 1 { + result.State = RawIdentityAmbiguous + return result, nil + } + for _, b := range cohorts { + copy := b + anchor = © + } + } + if anchor == nil { + return result, errors.New("raw identity cohort has no active anchor") + } + result.State = RawIdentityUnique + result.SessionID = anchor.Session + result.ContentRevision = anchor.Revision + result.ProjectionGeneration = anchor.Generation + result.PublicID = base + var aliases int + if err = tx.QueryRowContext(ctx, `SELECT count(DISTINCT a.group_id) FROM raw_session_public_aliases a JOIN sessions s ON s.raw_group_id=a.group_id WHERE a.alias_id=$1`, base).Scan(&aliases); err != nil { + return result, err + } + if len(cohorts) > 1 || aliases > 1 { + result.PublicID = base + "~" + cohorts[anchor.Session].ID + } + err = tx.QueryRowContext(ctx, `SELECT projection_generation FROM raw_source_projections WHERE source_id=$1`, anchor.Source).Scan(&result.SelectedGeneration) + return result, err +} +func rawGroupVariants(ctx context.Context, tx *sql.Tx, group string) ([]string, error) { + branches, err := loadRawBranches(ctx, tx, "group_id", group) + if err != nil { + return nil, err + } + overlays, err := loadRawOverlays(ctx, tx, group) + if err != nil { + return nil, err + } + var base string + if err = tx.QueryRowContext(ctx, `SELECT base_alias FROM raw_session_groups WHERE group_id=$1`, group).Scan(&base); err != nil { + return nil, err + } + seen := map[string]bool{} + var variants []string + for _, b := range branches { + if b.Active && !overlays.boolean(b.ID, "excluded") && !seen[b.Session] { + seen[b.Session] = true + variants = append(variants, base+"~"+b.ID) + } + } + slices.Sort(variants) + return variants, nil +} diff --git a/internal/postgres/raw_projection_indexes_pgtest_test.go b/internal/postgres/raw_projection_indexes_pgtest_test.go new file mode 100644 index 0000000000..d0f112c826 --- /dev/null +++ b/internal/postgres/raw_projection_indexes_pgtest_test.go @@ -0,0 +1,122 @@ +//go:build pgtest + +package postgres + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRawProjectionProofLookupsStayBounded(t *testing.T) { + f := newProjectionFixture(t) + m, receipt := f.accept(t, "device-a", "indexed-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("indexed"))) + identity, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + source := rawSourceID(m) + for _, size := range []int{32, 8192} { + seedRawUnrelatedProof(t, f, source, identity.SessionID, size) + _, err = f.admin.ExecContext(t.Context(), `ANALYZE session_sources; ANALYZE sessions; ANALYZE raw_session_branches`) + require.NoError(t, err) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + cases := []struct{ name, query, arg, column string }{ + {"source", rawSourceProofDeleteSQL, source, "source_id"}, + {"content", rawSourceProofAttachSQL, identity.SessionID, "session_id"}, + {"physical", rawSourceProofDetachSQL, identity.SessionID, "physical_session_id"}, + {"group", rawGroupContentsSQL, identity.GroupID, "raw_group_id"}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("%d/%s/%s", size, mode, tc.name), func(t *testing.T) { + plan := rawLookupPlan(t, f.runtime, mode, tc.query, tc.arg, size == 32) + var indexed bool + var visit func(map[string]any) + visit = func(node map[string]any) { + if cond, ok := node["Index Cond"].(string); ok && strings.Contains(cond, tc.column+" =") { + indexed = true + } + if removed, ok := node["Rows Removed by Filter"].(float64); ok { + assert.LessOrEqual(t, removed, float64(2), "unrelated proof scanned: %v", node) + } + if scans, ok := node["Plans"].([]any); ok { + for _, v := range scans { + visit(v.(map[string]any)) + } + } + } + visit(plan) + assert.True(t, indexed, "missing selective index predicate: %v", plan) + // Heap/index work stays bounded even with 256x more unrelated proof. + blocks, _ := plan["Shared Hit Blocks"].(float64) + reads, _ := plan["Shared Read Blocks"].(float64) + assert.Less(t, blocks+reads, float64(100), "changed-source lookup touched unrelated pages: %v", plan) + }) + } + } + // Exercise the same predicates through equal receipts and actual removal / + // reactivation while the unrelated population remains present. + m, receipt = f.accept(t, "device-a", fmt.Sprintf("equal-%d", size), receipt.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("indexed"))) + empty := projectionOutcome("unused") + empty.Outcome.Results = nil + m, receipt = f.accept(t, "device-a", fmt.Sprintf("remove-%d", size), receipt.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, empty)) + m, receipt = f.accept(t, "device-a", fmt.Sprintf("return-%d", size), receipt.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("indexed"))) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM session_sources`).Scan(&count)) + assert.Equal(t, size+1, count) + } +} + +func rawLookupPlan(t *testing.T, pg *sql.DB, mode, query, arg string, small bool) map[string]any { + t.Helper() + conn, err := pg.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + tx, err := conn.BeginTx(t.Context(), nil) + require.NoError(t, err) + defer tx.Rollback() + _, err = tx.ExecContext(t.Context(), `SET LOCAL plan_cache_mode=`+mode) + require.NoError(t, err) + if small { + // A sequential scan is reasonable on the tiny fixture. Prove an index + // path exists there, but let the large fixture choose its normal plan. + _, err = tx.ExecContext(t.Context(), `SET LOCAL enable_seqscan=off`) + require.NoError(t, err) + } + _, err = tx.ExecContext(t.Context(), `PREPARE raw_lookup(text) AS `+query) + require.NoError(t, err) + var data []byte + require.NoError(t, tx.QueryRowContext(t.Context(), `EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) EXECUTE raw_lookup('`+strings.ReplaceAll(arg, "'", "''")+`')`).Scan(&data)) + _, err = tx.ExecContext(t.Context(), `DEALLOCATE raw_lookup`) + require.NoError(t, err) + var result []struct{ Plan map[string]any } + require.NoError(t, json.Unmarshal(data, &result)) + require.Len(t, result, 1) + return result[0].Plan +} + +func seedRawUnrelatedProof(t *testing.T, f projectionFixture, source, id string, size int) { + t.Helper() + statements := []string{ + `INSERT INTO raw_source_projections(source_id,device_id,provider,configured_root_id,source_key_sha256,selected_manifest_id,processing_version,projection_generation,selected_job_id) SELECT 'filler-source-'||n,p.device_id,p.provider,p.configured_root_id,p.source_key_sha256,p.selected_manifest_id,p.processing_version,p.projection_generation,p.selected_job_id FROM raw_source_projections p CROSS JOIN generate_series(1,$3::int)n WHERE p.source_id=$1 ON CONFLICT DO NOTHING`, + `INSERT INTO raw_session_groups(group_id,provider,logical_key,base_alias) SELECT 'filler-group-'||n,'codex','filler-key-'||n,'filler-alias-'||n FROM generate_series(1,$3::int)n ON CONFLICT DO NOTHING`, + `INSERT INTO raw_content_revisions(session_id,group_id,content_revision,payload) SELECT 'filler-row-'||n,'filler-group-'||n,'filler-revision-'||n,p.payload FROM raw_content_revisions p CROSS JOIN generate_series(1,$3::int)n WHERE p.session_id=$2 ON CONFLICT DO NOTHING`, + `INSERT INTO sessions(id,project,machine,agent,raw_group_id) SELECT 'filler-row-'||n,'synthetic','fixture','codex','filler-group-'||n FROM generate_series(1,$3::int)n ON CONFLICT DO NOTHING`, + `INSERT INTO raw_session_branches(branch_id,source_id,group_id,member_id,session_id,content_revision,manifest_id,processing_version,projection_generation,active,prior_payload) SELECT 'filler-branch-'||n,'filler-source-'||n,'filler-group-'||n,'filler-member-'||n,'filler-row-'||n,'filler-revision-'||n,p.selected_manifest_id,p.processing_version,p.projection_generation,true,''::bytea FROM raw_source_projections p CROSS JOIN generate_series(1,$3::int)n WHERE p.source_id=$1 ON CONFLICT DO NOTHING`, + `INSERT INTO session_sources(branch_id,group_id,source_id,session_id,physical_session_id,manifest_id,content_revision,processing_version,projection_generation) SELECT branch_id,group_id,source_id,session_id,session_id,manifest_id,content_revision,processing_version,projection_generation FROM raw_session_branches WHERE branch_id LIKE 'filler-branch-%' ON CONFLICT DO NOTHING`, + } + for _, statement := range statements { + // Explicit parameter casts keep unused positions well-typed in statements + // which need only one of the fixture keys. + statement = `WITH fixture_parameters AS (SELECT $1::text,$2::text,$3::int) ` + statement + _, err := f.runtime.ExecContext(t.Context(), statement, source, id, size) + require.NoError(t, err) + } +} diff --git a/internal/postgres/raw_projection_links.go b/internal/postgres/raw_projection_links.go new file mode 100644 index 0000000000..49bfda8385 --- /dev/null +++ b/internal/postgres/raw_projection_links.go @@ -0,0 +1,94 @@ +package postgres + +import ( + "context" + "database/sql" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" +) + +const rawLinksDDL = `CREATE TABLE IF NOT EXISTS raw_session_links ( + branch_id TEXT NOT NULL,kind TEXT NOT NULL,ordinal INTEGER NOT NULL,call_index INTEGER NOT NULL,event_index INTEGER NOT NULL,target_alias TEXT NOT NULL, + PRIMARY KEY(branch_id,kind,ordinal,call_index,event_index) +);` + +// Only source-owned typed relationships are recorded. Missing source proof +// cannot redirect a link to a different device's equally named session. +func writeRawLinks(ctx context.Context, tx *sql.Tx, branch string, p ingest.PreparedSession) error { + if _, err := tx.ExecContext(ctx, `DELETE FROM raw_session_links WHERE branch_id=$1`, branch); err != nil { + return err + } + add := func(kind string, ordinal, call, event int, target string) error { + if target == "" { + return nil + } + _, err := tx.ExecContext(ctx, `INSERT INTO raw_session_links(branch_id,kind,ordinal,call_index,event_index,target_alias) VALUES($1,$2,$3,$4,$5,$6)`, branch, kind, ordinal, call, event, target) + return err + } + if p.Session.ParentSessionID != nil { + if err := add("parent", -1, -1, -1, *p.Session.ParentSessionID); err != nil { + return err + } + } + for _, m := range p.Messages { + for ci, c := range m.ToolCalls { + if err := add("call", m.Ordinal, ci, -1, c.SubagentSessionID); err != nil { + return err + } + for ei, e := range c.ResultEvents { + if err := add("event", m.Ordinal, ci, ei, e.SubagentSessionID); err != nil { + return err + } + } + } + } + return nil +} + +// Exact-source authority survives removal and exclusion. Only an edge without +// that historical authority may use a unique materialized content cohort from +// the same immutable capture scope. Count per owner edge before merging owners: +// equal proofs coalesce, while independently proven plural parents survive. +const hostedLinkFromSQL = ` FROM raw_session_links e + JOIN raw_session_branches owner ON owner.branch_id=e.branch_id AND owner.active + JOIN sessions own_session ON own_session.id=owner.session_id + JOIN raw_source_projections owner_source ON owner_source.source_id=owner.source_id + JOIN LATERAL ( + SELECT EXISTS ( + SELECT 1 FROM raw_session_public_aliases a + JOIN raw_session_branches b ON b.group_id=a.group_id AND b.source_id=owner.source_id + AND (a.anchor_branch='' OR a.anchor_branch=b.branch_id) + WHERE a.alias_id=e.target_alias + ) AS exact + ) authority ON true + JOIN LATERAL ( + SELECT candidates.session_id,count(*) OVER () AS cohort_count FROM ( + SELECT DISTINCT b.session_id + FROM raw_session_public_aliases a + JOIN raw_session_branches b ON b.group_id=a.group_id AND b.active + AND (a.anchor_branch='' OR a.anchor_branch=b.branch_id) + JOIN raw_source_projections target_source ON target_source.source_id=b.source_id + JOIN sessions materialized ON materialized.id=b.session_id + WHERE a.alias_id=e.target_alias + AND (b.source_id=owner.source_id OR (NOT authority.exact + AND target_source.tenant_id=owner_source.tenant_id + AND target_source.device_id=owner_source.device_id + AND target_source.provider=owner_source.provider + AND target_source.configured_root_id=owner_source.configured_root_id)) + AND NOT COALESCE((SELECT c.value::boolean FROM raw_curation c + WHERE c.group_id=b.group_id AND c.field='excluded' AND c.branch_id IN ('',b.branch_id) + ORDER BY c.branch_id DESC LIMIT 1),false) + ) candidates + ) target ON authority.exact OR target.cohort_count=1 + JOIN sessions target_session ON target_session.id=target.session_id + WHERE NOT COALESCE((SELECT c.value::boolean FROM raw_curation c WHERE c.group_id=owner.group_id AND c.field='excluded' AND c.branch_id IN ('',owner.branch_id) ORDER BY c.branch_id DESC LIMIT 1),false) ` + +func (s *Store) sessionDialect() db.QueryDialect { + d := db.PostgresQueryDialect() + if !s.hostedRelations { + return d + } + return d.WithParentRelation(func(child, parent string) string { + return "((" + child + ".provenance_kind='legacy' AND " + child + ".parent_session_id=" + parent + ".id) OR (" + child + ".provenance_kind='raw' AND EXISTS(SELECT 1 " + hostedLinkFromSQL + " AND e.kind='parent' AND owner.session_id=" + child + ".id AND target.session_id=" + parent + ".id)))" + }) +} diff --git a/internal/postgres/raw_projection_links_pgtest_test.go b/internal/postgres/raw_projection_links_pgtest_test.go new file mode 100644 index 0000000000..fdcf3e1429 --- /dev/null +++ b/internal/postgres/raw_projection_links_pgtest_test.go @@ -0,0 +1,352 @@ +//go:build pgtest + +package postgres + +import ( + "fmt" + "go.kenn.io/agentsview/internal/db" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" +) + +func linkChildOutcome() rawderive.ParsedManifest { + p := projectionOutcome("child") + p.Outcome.Results[0].Result.Session.ID = "codex:child" + p.Outcome.Results[0].Result.Session.SourceSessionID = "child" + p.Outcome.Results[0].Result.Session.ParentSessionID = "codex:portable" + p.Outcome.Results[0].Result.Session.RelationshipType = "subagent" + p.Outcome.Results[0].Result.Session.EndedAt = p.Outcome.Results[0].Result.Session.StartedAt.Add(7 * time.Second) + return p +} + +// The fallback is bounded to the immutable capture scope, and counts content +// cohorts rather than duplicate proofs. Public reads must not choose a variant. +func TestHostedCrossSourceRelationshipScope(t *testing.T) { + for _, mode := range []string{"unique", "equal_cohort", "ambiguous", "other_device", "other_root", "other_provider"} { + t.Run(mode, func(t *testing.T) { + f := newProjectionFixture(t) + child, _ := f.acceptScoped(t, "device-a", "child", "", parser.AgentCodex, "root-a", "child.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, child), child, linkChildOutcome())) + device, root, provider := "device-a", "root-a", parser.AgentCodex + switch mode { + case "other_device": + device = "device-b" + case "other_root": + root = "root-b" + case "other_provider": + provider = parser.AgentClaude + } + parent, _ := f.acceptScoped(t, device, "parent", "", provider, root, "parent.jsonl") + p := projectionOutcome("parent") + p.Outcome.Results[0].Result.Session.Agent = provider + if mode == "unique" { + call := &p.Outcome.Results[0].Result.Messages[1].ToolCalls[0] + call.SubagentSessionID = "codex:child" + call.ResultEvents = []parser.ParsedToolResultEvent{{ToolUseID: "call-1", SubagentSessionID: "codex:child", Source: "tool_result", Status: "completed", Content: "child complete"}} + p.Outcome.Results[0].Result.Messages[1].HasToolUse = true + } + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, parent), parent, p)) + if mode == "equal_cohort" || mode == "ambiguous" { + second, _ := f.acceptScoped(t, "device-a", "second", "", parser.AgentCodex, "root-a", "second.jsonl") + text := "parent" + if mode == "ambiguous" { + text = "different parent" + } + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, second), second, projectionOutcome(text))) + } + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + got, err := h.GetSessionFull(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, got) + if mode == "unique" || mode == "equal_cohort" { + require.NotNil(t, got.ParentSessionID) + assert.Equal(t, "codex:portable", *got.ParentSessionID) + children, err := h.GetChildSessions(t.Context(), "codex:portable") + require.NoError(t, err) + require.Len(t, children, 1) + assert.Equal(t, "codex:child", children[0].ID) + if mode == "unique" { + messages, err := h.GetAllMessages(t.Context(), "codex:portable") + require.NoError(t, err) + require.Len(t, messages, 2) + require.Len(t, messages[1].ToolCalls, 1) + call := messages[1].ToolCalls[0] + assert.Equal(t, "codex:child", call.SubagentSessionID) + require.Len(t, call.ResultEvents, 1) + assert.Equal(t, "codex:child", call.ResultEvents[0].SubagentSessionID) + timing, err := h.GetSessionTiming(t.Context(), "codex:portable") + require.NoError(t, err) + require.NotNil(t, timing.SlowestCall) + require.NotNil(t, timing.SlowestCall.DurationMs) + assert.Equal(t, int64(7000), *timing.SlowestCall.DurationMs) + sidebar, err := h.GetSidebarSessionIndex(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + assert.Equal(t, 1, sidebar.Total) + require.Len(t, sidebar.Sessions, 2) + } + } else { + assert.Nil(t, got.ParentSessionID) + assert.Empty(t, got.ParentSessionIDs) + } + }) + } +} + +func TestHostedCrossSourceHistoricalExactAuthority(t *testing.T) { + for _, mode := range []string{"removed", "excluded"} { + t.Run(mode, func(t *testing.T) { + f := newProjectionFixture(t) + own, accepted := f.acceptScoped(t, "device-a", "own", "", parser.AgentCodex, "root-a", "own.jsonl") + combined := projectionOutcome("exact parent") + combined.Outcome.Results = append(combined.Outcome.Results, linkChildOutcome().Outcome.Results...) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, own), own, combined)) + if mode == "removed" { + next, _ := f.acceptScoped(t, "device-a", "removed", accepted.Receipt, parser.AgentCodex, "root-a", "own.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, linkChildOutcome())) + } else { + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "trashed", true)) + removed, err := f.sink.ExcludeTrashedSession(t.Context(), "codex:portable") + require.NoError(t, err) + require.True(t, removed) + } + // A different logical group advertises the same explicit alias. Its raw + // capture tuple is compatible, but it cannot defeat historical authority in + // another alias group. This deliberately exercises the broader alias set. + alternate, _ := f.acceptScoped(t, "device-a", "alternate", "", parser.AgentCodex, "root-a", "alternate.jsonl") + p := projectionOutcome("fallback parent") + p.Outcome.Results[0].Result.Session.SourceSessionID = "alternate-key" + p.Outcome.Results[0].Result.Session.StartedAt = time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, alternate), alternate, p)) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + got, err := h.GetSessionFull(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, got) + assert.Nil(t, got.ParentSessionID) + assert.Empty(t, got.ParentSessionIDs) + }) + } +} + +func TestHostedCrossSourceAliasAnchorLimitsHistoricalAuthority(t *testing.T) { + f := newProjectionFixture(t) + own, accepted := f.acceptScoped(t, "device-a", "own", "", parser.AgentCodex, "root-a", "own.jsonl") + combined := projectionOutcome("historical parent") + combined.Outcome.Results = append(combined.Outcome.Results, linkChildOutcome().Outcome.Results...) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, own), own, combined)) + alternate, _ := f.acceptScoped(t, "device-a", "alternate", "", parser.AgentCodex, "root-a", "alternate.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, alternate), alternate, projectionOutcome("selected parent"))) + anchor := f.alias(t, alternate) + child := linkChildOutcome() + child.Outcome.Results[0].Result.Session.ParentSessionID = anchor + next, _ := f.acceptScoped(t, "device-a", "removed", accepted.Receipt, parser.AgentCodex, "root-a", "own.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, child)) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + got, err := h.GetSessionFull(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, got.ParentSessionID) + assert.Equal(t, "codex:portable", *got.ParentSessionID) + messages, err := h.GetAllMessages(t.Context(), *got.ParentSessionID) + require.NoError(t, err) + require.NotEmpty(t, messages) + assert.Equal(t, "selected parent", messages[0].Content) +} + +func TestHostedCrossSourceLinkLookupsStayBounded(t *testing.T) { + f := newProjectionFixture(t) + child, _ := f.acceptScoped(t, "device-a", "child", "", parser.AgentCodex, "root-a", "child.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, child), child, linkChildOutcome())) + parent, _ := f.acceptScoped(t, "device-a", "parent", "", parser.AgentCodex, "root-a", "parent.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, parent), parent, projectionOutcome("parent"))) + identity, err := f.sink.Resolve(t.Context(), "codex:child") + require.NoError(t, err) + for _, size := range []int{32, 8192} { + seedRawUnrelatedProof(t, f, rawSourceID(child), identity.SessionID, size) + _, err = f.runtime.ExecContext(t.Context(), `INSERT INTO raw_session_public_aliases(alias_id,group_id) SELECT base_alias,group_id FROM raw_session_groups WHERE group_id LIKE 'filler-group-%' ON CONFLICT DO NOTHING; + INSERT INTO raw_session_links(branch_id,kind,ordinal,call_index,event_index,target_alias) SELECT branch_id,'parent',-1,-1,-1,'filler-alias-'||substr(branch_id,15) FROM raw_session_branches WHERE branch_id LIKE 'filler-branch-%' ON CONFLICT DO NOTHING`) + require.NoError(t, err) + _, err = f.admin.ExecContext(t.Context(), `ANALYZE sessions; ANALYZE raw_session_links; ANALYZE raw_session_branches; ANALYZE raw_source_projections; ANALYZE raw_session_public_aliases`) + require.NoError(t, err) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + t.Run(fmt.Sprintf("%d/%s", size, mode), func(t *testing.T) { + plan := rawLookupPlan(t, f.runtime, mode, `SELECT DISTINCT target.session_id `+hostedLinkFromSQL+` AND owner.session_id=$1`, identity.SessionID, size == 32) + assert.Equal(t, float64(1), plan["Actual Rows"]) + buffers := plan["Shared Hit Blocks"].(float64) + plan["Shared Read Blocks"].(float64) + assert.Less(t, buffers, float64(200), "targeted relation touches unrelated pages: %v", plan) + indexedAlias := false + var visit func(map[string]any) + visit = func(node map[string]any) { + if cond, ok := node["Index Cond"].(string); ok && strings.Contains(cond, "alias_id =") { + indexedAlias = true + } + if removed, ok := node["Rows Removed by Filter"].(float64); ok { + assert.LessOrEqual(t, removed, float64(3), "unrelated proof scanned: %v", node) + } + if children, ok := node["Plans"].([]any); ok { + for _, c := range children { + visit(c.(map[string]any)) + } + } + } + visit(plan) + assert.True(t, indexedAlias, "alias must constrain candidate search") + t.Logf("size=%d %s shared buffers=%.0f", size, mode, buffers) + }) + } + } +} + +func TestHostedCrossSourcePluralOwnersRemainContextual(t *testing.T) { + f := newProjectionFixture(t) + for _, device := range []string{"device-a", "device-b"} { + child, _ := f.acceptScoped(t, device, "child", "", parser.AgentCodex, "root-a", "child.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, child), child, linkChildOutcome())) + parent, _ := f.acceptScoped(t, device, "parent", "", parser.AgentCodex, "root-a", "parent.jsonl") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, parent), parent, projectionOutcome(device))) + } + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + _, err = h.GetSessionFull(t.Context(), "codex:portable") + var conflict *db.SessionIdentityError + require.ErrorAs(t, err, &conflict) + require.Len(t, conflict.Variants, 2) + child, err := h.GetSessionFull(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, child) + assert.Nil(t, child.ParentSessionID) + assert.Equal(t, conflict.Variants, child.ParentSessionIDs) + for _, parent := range conflict.Variants { + children, err := h.GetChildSessions(t.Context(), parent) + require.NoError(t, err) + require.Len(t, children, 1) + require.NotNil(t, children[0].ParentSessionID) + assert.Equal(t, parent, *children[0].ParentSessionID) + } +} + +// The owner is published once. Only its separately captured target changes, +// including a target revision between physical reads and public edge mapping. +func TestHostedCrossSourceTargetLifecycleAndReadRevision(t *testing.T) { + f := newProjectionFixture(t) + own, _ := f.acceptScoped(t, "device-a", "owner", "", parser.AgentCodex, "root-a", "child.jsonl") + child := linkChildOutcome() + child.Outcome.Results[0].Result.Messages[1].HasToolUse = true + child.Outcome.Results[0].Result.Messages[1].ToolCalls[0].SubagentSessionID = "codex:portable" + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, own), own, child)) + h, err := newHostedAdapter(f.runtime, f.tenant) + require.NoError(t, err) + ownerIdentity, err := f.sink.Resolve(t.Context(), "codex:child") + require.NoError(t, err) + receipt := "" + publish := func(capture string, seconds int) { + target, accepted := f.acceptScoped(t, "device-a", capture, receipt, parser.AgentCodex, "root-a", "parent.jsonl") + receipt = accepted.Receipt + result := projectionOutcome("parent") + result.Outcome.Results[0].Result.Session.EndedAt = result.Outcome.Results[0].Result.Session.StartedAt.Add(time.Duration(seconds) * time.Second) + if seconds == 0 { + result.Outcome.Results = nil + } + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, target), target, result)) + } + check := func(present bool, seconds int) { + got, err := h.GetSessionFull(t.Context(), "codex:child") + require.NoError(t, err) + require.NotNil(t, got) + if present { + require.NotNil(t, got.ParentSessionID) + assert.Equal(t, "codex:portable", *got.ParentSessionID) + } else { + assert.Nil(t, got.ParentSessionID) + } + children, err := h.GetChildSessions(t.Context(), "codex:portable") + require.NoError(t, err) + if present { + require.Len(t, children, 1) + assert.Equal(t, "codex:child", children[0].ID) + } else { + assert.Empty(t, children) + } + messages, err := h.GetAllMessages(t.Context(), "codex:child") + require.NoError(t, err) + require.Len(t, messages, 2) + require.Len(t, messages[1].ToolCalls, 1) + expected := "" + if present { + expected = "codex:portable" + } + assert.Equal(t, expected, messages[1].ToolCalls[0].SubagentSessionID) + timing, err := h.GetSessionTiming(t.Context(), "codex:child") + require.NoError(t, err) + if present { + require.NotNil(t, timing.SlowestCall) + require.NotNil(t, timing.SlowestCall.DurationMs) + assert.Equal(t, int64(seconds*1000), *timing.SlowestCall.DurationMs) + } else { + assert.Nil(t, timing.SlowestCall) + require.Len(t, timing.Turns, 1) + require.Len(t, timing.Turns[0].Calls, 1) + assert.Nil(t, timing.Turns[0].Calls[0].DurationMs) + } + sidebar, err := h.GetSidebarSessionIndex(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + assert.Equal(t, 1, sidebar.Total) + count := 1 + if present { + count = 2 + } + require.Len(t, sidebar.Sessions, count) + var ownerSession, ownerManifest string + var generation int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT session_id,manifest_id,projection_generation FROM raw_session_branches WHERE source_id=$1`, rawSourceID(own)).Scan(&ownerSession, &ownerManifest, &generation)) + assert.Equal(t, ownerIdentity.SessionID, ownerSession) + assert.Equal(t, own.ManifestID, ownerManifest) + assert.Equal(t, 1, generation) + } + check(false, 0) + publish("target", 7) + check(true, 7) + publish("removed", 0) + check(false, 0) + publish("reintroduced", 7) + check(true, 7) + type snapshot struct { + session *db.Session + duration int64 + } + reads := 0 + result, err := hostedRead(t.Context(), h, func(_ hostedRevision) (snapshot, error) { + reads++ + physical, err := h.physical.GetSessionFull(t.Context(), ownerIdentity.SessionID) + if err != nil { + return snapshot{}, err + } + timing, err := h.GetSessionTiming(t.Context(), "codex:child") + if err != nil { + return snapshot{}, err + } + require.NotNil(t, timing.SlowestCall) + require.NotNil(t, timing.SlowestCall.DurationMs) + duration := *timing.SlowestCall.DurationMs + if reads == 1 { + publish("mid-read-target", 13) + } + refs := hostedRefs{} + refs.session(physical) + err = refs.mapIDs(t.Context(), h) + return snapshot{physical, duration}, err + }) + require.NoError(t, err) + assert.Equal(t, 2, reads) + assert.Equal(t, int64(13000), result.duration) + require.NotNil(t, result.session.ParentSessionID) + assert.Equal(t, "codex:portable", *result.session.ParentSessionID) + check(true, 13) +} diff --git a/internal/postgres/raw_projection_pgtest_test.go b/internal/postgres/raw_projection_pgtest_test.go new file mode 100644 index 0000000000..a2150a5cf5 --- /dev/null +++ b/internal/postgres/raw_projection_pgtest_test.go @@ -0,0 +1,310 @@ +//go:build pgtest + +package postgres + +import ( + "bytes" + "crypto/sha256" + "fmt" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +type projectionFixture struct { + hostedFixture + sink *RawProjectionStore + custody *rawsync.Service + jobs *RawIngestStore + objects rawsync.ObjectStore +} + +func newProjectionFixture(t *testing.T) projectionFixture { + t.Helper() + f := newHostedFixture(t, "tenant-projection") + repo, err := artifact.OpenRepository(t.Context(), t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, repo.Close()) }) + objects, err := rawsync.NewArtifactObjectStore(repo.Content()) + require.NoError(t, err) + jobs, err := NewTenantRawIngestStore(f.runtime, f.tenant) + require.NoError(t, err) + custody, err := rawsync.NewService(objects, jobs, rawsync.DefaultManifestLimits(), "parser-1") + require.NoError(t, err) + sink, err := NewRawProjectionStore(f.runtime, RawProjectionOptions{Tenant: f.tenant}) + require.NoError(t, err) + return projectionFixture{f, sink, custody, jobs, objects} +} +func (f projectionFixture) accept(t *testing.T, device, capture, parent string, providers ...parser.AgentType) (rawsync.CanonicalManifest, rawsync.CommitResult) { + t.Helper() + provider := parser.AgentCodex + if len(providers) > 0 { + provider = providers[0] + } + return f.acceptScoped(t, device, capture, parent, provider, "", "") +} + +func (f projectionFixture) acceptScoped(t *testing.T, device, capture, parent string, provider parser.AgentType, root, source string) (rawsync.CanonicalManifest, rawsync.CommitResult) { + t.Helper() + digest := sha256.Sum256([]byte("synthetic-" + device)) + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO raw_devices(device_id,display_name,credential_sha256,created_at) VALUES($1,$1,$2,clock_timestamp()) ON CONFLICT(device_id) DO NOTHING`, device, digest[:]) + require.NoError(t, err) + identity, err := rawsync.NewAuthIdentity(f.tenant, device) + require.NoError(t, err) + body := []byte("synthetic custody " + capture) + ref := rawCustodyObjectRef(t, body) + _, err = f.custody.FinalizeObject(t.Context(), identity, provider, ref, bytes.NewReader(body)) + require.NoError(t, err) + manifest := rawCustodyManifest(capture, parent, rawIngestCapturedAt(), ref) + manifest.Provider = provider + if root != "" { + manifest.ConfiguredRootID = root + } + if source != "" { + manifest.SourceKey = source + } + canonical, err := rawsync.ValidateAndCanonicalize(identity, manifest, rawsync.DefaultManifestLimits()) + require.NoError(t, err) + result, err := f.custody.CommitManifest(t.Context(), identity, manifest) + require.NoError(t, err) + return canonical, result +} +func projectionOutcome(content string) rawderive.ParsedManifest { + return rawderive.ParsedManifest{Outcome: parser.ParseOutcome{ResultSetComplete: true, Results: []parser.ParseResultOutcome{{Result: parser.ParseResult{ + Session: parser.ParsedSession{ID: "codex:portable", SourceSessionID: "portable", Agent: parser.AgentCodex, Project: "synthetic", Machine: "capture-device", StartedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}, + Messages: []parser.ParsedMessage{{Ordinal: 0, Role: parser.RoleUser, Content: content}, {Ordinal: 1, Role: parser.RoleAssistant, Content: "checking", ToolCalls: []parser.ParsedToolCall{{ToolUseID: "call-1", ToolName: "Bash", Category: "Bash", InputJSON: `{"command":"true"}`}}}}, + UsageEvents: []parser.ParsedUsageEvent{{Source: "turn", Model: "synthetic-model", InputTokens: 11, OutputTokens: 7}}, + }}}}} +} +func (f projectionFixture) lease(t *testing.T, m rawsync.CanonicalManifest) rawderive.JobLease { + t.Helper() + _, err := f.sink.SelectSourceGeneration(t.Context(), m, "parser-1") + require.NoError(t, err) + leases, err := f.jobs.ClaimRawParseJobs(t.Context(), "projection-worker", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + return leases[0] +} + +// Losing the atomic write/completion boundary or any normalized child write +// must fail this through actual custody, lease, and normal physical Store reads. +func TestRawProjectionPublishesNormalizedGraph(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + lease := f.lease(t, m) + require.NoError(t, f.sink.Project(t.Context(), lease, m, projectionOutcome("hello"))) + resolved, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + require.Equal(t, RawIdentityUnique, resolved.State) + store := &Store{pg: f.runtime} + messages, err := store.GetMessages(t.Context(), resolved.SessionID, 0, 10, true) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "hello", messages[0].Content) + require.Len(t, messages[1].ToolCalls, 1) + assert.Equal(t, "Bash", messages[1].ToolCalls[0].ToolName) + events, err := store.GetSessionUsageRows(t.Context(), []string{resolved.SessionID}) + require.NoError(t, err) + require.NotNil(t, events) + require.Len(t, events.Rows, 1) + assert.Equal(t, 11, events.Rows[0].InputTokens) + assert.Equal(t, 7, events.Rows[0].OutputTokens) + var state string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE id=$1`, lease.ID).Scan(&state)) + assert.Equal(t, "complete", state) + assert.ErrorIs(t, f.sink.Project(t.Context(), lease, m, projectionOutcome("must not replace")), rawderive.ErrLeaseLost) +} + +func TestRawProjectionGenerationAndLeaseFences(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + old := f.lease(t, m) + generation, err := f.sink.SelectSourceGeneration(t.Context(), m, "parser-2") + require.NoError(t, err) + assert.Greater(t, generation, old.ProjectionGeneration) + assert.ErrorIs(t, f.sink.Project(t.Context(), old, m, projectionOutcome("old")), rawderive.ErrLeaseLost) + leases, err := f.jobs.ClaimRawParseJobs(t.Context(), "successor", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + require.NoError(t, f.sink.Project(t.Context(), leases[0], m, projectionOutcome("new"))) + again, err := f.sink.SelectSourceGeneration(t.Context(), m, "parser-2") + require.NoError(t, err) + assert.Equal(t, generation, again) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_projection_generations`).Scan(&count)) + assert.Equal(t, 2, count) + resolved, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + messages, err := (&Store{pg: f.runtime}).GetMessages(t.Context(), resolved.SessionID, 0, 10, true) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "new", messages[0].Content) +} +func TestRawProjectionExpiredAndSupersededLeases(t *testing.T) { + for _, mode := range []string{"expired", "superseded"} { + t.Run(mode, func(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "capture-a", "") + lease := f.lease(t, m) + if mode == "expired" { + _, err := f.runtime.ExecContext(t.Context(), `UPDATE raw_ingest_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1`, lease.ID) + require.NoError(t, err) + } else { + f.accept(t, "device-a", "capture-b", accepted.Receipt) + } + assert.ErrorIs(t, f.sink.Project(t.Context(), lease, m, projectionOutcome("old")), rawderive.ErrLeaseLost) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Zero(t, count) + }) + } +} +func TestRawProjectionRollsBackPayloadAndProofWhenCompletionFails(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + lease := f.lease(t, m) + _, err := f.admin.ExecContext(t.Context(), `CREATE FUNCTION reject_projection_completion() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.state='complete' THEN RAISE EXCEPTION 'synthetic completion dependency unavailable'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_projection_completion BEFORE UPDATE ON raw_ingest_jobs FOR EACH ROW EXECUTE FUNCTION reject_projection_completion()`) + require.NoError(t, err) + err = f.sink.Project(t.Context(), lease, m, projectionOutcome("rollback")) + require.Error(t, err) + assert.Contains(t, err.Error(), "synthetic completion dependency unavailable") + for _, table := range []string{"sessions", "raw_session_branches", "raw_content_revisions", "raw_source_contributions", "raw_embedding_outbox"} { + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM `+table).Scan(&count)) + assert.Zero(t, count, table) + } + var revision int64 + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT corpus_revision FROM raw_corpus_state`).Scan(&revision)) + assert.Zero(t, revision) + var state string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE id=$1`, lease.ID).Scan(&state)) + assert.Equal(t, "leased", state) +} + +func (f projectionFixture) alias(t *testing.T, m rawsync.CanonicalManifest) string { + t.Helper() + var alias string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT a.alias_id FROM raw_session_public_aliases a JOIN raw_session_branches b ON b.branch_id=a.anchor_branch WHERE b.source_id=$1`, rawSourceID(m)).Scan(&alias)) + return alias +} +func TestRawProjectionCohortCurationFollowsAliases(t *testing.T) { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprint(reverse), func(t *testing.T) { + f := newProjectionFixture(t) + devices := []string{"device-a", "device-b"} + if reverse { + slices.Reverse(devices) + } + manifests := map[string]rawsync.CanonicalManifest{} + receipts := map[string]string{} + for _, device := range devices { + m, accepted := f.accept(t, device, "capture-"+device, "") + manifests[device] = m + receipts[device] = accepted.Receipt + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome(device))) + } + a := f.alias(t, manifests["device-a"]) + b := f.alias(t, manifests["device-b"]) + require.NoError(t, f.sink.SetCuration(t.Context(), a, "trashed", true)) + require.NoError(t, f.sink.SetCuration(t.Context(), b, "starred", true)) + require.NoError(t, f.sink.SetCuration(t.Context(), a, "display_name", "alpha")) + require.NoError(t, f.sink.SetCuration(t.Context(), b, "display_name", "beta")) + // Convergence cannot overwrite either branch's choices. The visible branch + // supplies the name and star; the hidden duplicate is not a trash-list item. + m, accepted := f.accept(t, "device-a", "converged-a", receipts["device-a"]) + receipts["device-a"] = accepted.Receipt + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("device-b"))) + ar, err := f.sink.Resolve(t.Context(), a) + require.NoError(t, err) + br, err := f.sink.Resolve(t.Context(), b) + require.NoError(t, err) + assert.Equal(t, ar.SessionID, br.SessionID) + var name string + var trashed, starred bool + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT display_name,deleted_at IS NOT NULL,EXISTS(SELECT 1 FROM starred_sessions WHERE session_id=s.id) FROM sessions s WHERE id=$1`, ar.SessionID).Scan(&name, &trashed, &starred)) + assert.Equal(t, "beta", name) + assert.False(t, trashed) + assert.True(t, starred) + require.NoError(t, f.sink.SetCuration(t.Context(), a, "starred", false)) + require.NoError(t, f.sink.SetCuration(t.Context(), a, "trashed", true)) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT deleted_at IS NOT NULL,EXISTS(SELECT 1 FROM starred_sessions WHERE session_id=s.id) FROM sessions s WHERE id=$1`, ar.SessionID).Scan(&trashed, &starred)) + assert.True(t, trashed) + assert.False(t, starred) + require.NoError(t, f.sink.SetCuration(t.Context(), b, "trashed", false)) + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "display_name", "shared")) + m, _ = f.accept(t, "device-a", "split-a", receipts["device-a"]) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("split"))) + for _, alias := range []string{a, b} { + r, err := f.sink.Resolve(t.Context(), alias) + require.NoError(t, err) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT display_name,deleted_at IS NOT NULL,EXISTS(SELECT 1 FROM starred_sessions WHERE session_id=s.id) FROM sessions s WHERE id=$1`, r.SessionID).Scan(&name, &trashed, &starred)) + assert.Equal(t, "shared", name) + assert.False(t, trashed) + assert.False(t, starred) + } + r, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityAmbiguous, r.State) + require.Len(t, r.Variants, 2) + }) + } +} + +func TestRawProjectionPinsRemainUnresolvedAfterOrdinalReuse(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "pin-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("original"))) + alias := f.alias(t, m) + require.NoError(t, f.sink.SetPin(t.Context(), "codex:portable", 0, true, "group note")) + require.NoError(t, f.sink.SetPin(t.Context(), alias, 1, true, "other pin")) + before, err := f.sink.Resolve(t.Context(), alias) + require.NoError(t, err) + pins, err := (&Store{pg: f.runtime}).ListPinnedMessages(t.Context(), before.SessionID, "") + require.NoError(t, err) + require.Len(t, pins, 2) + m, _ = f.accept(t, "device-a", "pin-b", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("replacement"))) + after, err := f.sink.Resolve(t.Context(), alias) + require.NoError(t, err) + pins, err = (&Store{pg: f.runtime}).ListPinnedMessages(t.Context(), after.SessionID, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + refs, err := f.sink.PinReferences(t.Context(), alias) + require.NoError(t, err) + require.Len(t, refs, 2) + var unresolved int + for _, ref := range refs { + if !ref.Resolved { + unresolved++ + assert.Equal(t, 0, ref.Ordinal) + assert.Equal(t, "group note", ref.Note) + } + } + assert.Equal(t, 1, unresolved) + require.NoError(t, f.sink.SetPin(t.Context(), alias, 1, false, "")) + pins, err = (&Store{pg: f.runtime}).ListPinnedMessages(t.Context(), after.SessionID, "") + require.NoError(t, err) + assert.Empty(t, pins) +} + +func TestRawProjectionClaimsNeverResurrectUnselectedGeneration(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "selection-claim", "") + old := f.lease(t, m) + _, err := f.runtime.ExecContext(t.Context(), `UPDATE raw_ingest_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1`, old.ID) + require.NoError(t, err) + _, err = f.sink.SelectSourceGeneration(t.Context(), m, "parser-2") + require.NoError(t, err) + leases, err := f.jobs.ClaimRawParseJobs(t.Context(), "selected-only", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + assert.Equal(t, "parser-2", leases[0].ProcessingVersion) +} diff --git a/internal/postgres/raw_projection_pin_batch.go b/internal/postgres/raw_projection_pin_batch.go new file mode 100644 index 0000000000..72576b6904 --- /dev/null +++ b/internal/postgres/raw_projection_pin_batch.go @@ -0,0 +1,143 @@ +package postgres + +import ( + "context" + "database/sql" + "slices" + "time" + + "go.kenn.io/agentsview/internal/db" +) + +// Effective pin selection uses the same precedence as selectRawEffectivePins: +// visible eligible cohort members, explicit branch values before group defaults, +// any true member pins the cohort, first lexical explicit member supplies note. +// False overrides suppress their own inherited value before page selection. +const rawEffectivePinCandidatesSQL = `WITH eligible AS ( + SELECT b.branch_id,b.group_id,b.session_id, + COALESCE((SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id=b.branch_id AND c.field='trashed'),(SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id='' AND c.field='trashed'),false) trashed + FROM raw_session_branches b JOIN sessions s ON s.id=b.session_id + WHERE b.active AND ($1='' OR s.id=$1) AND ($1<>'' OR s.deleted_at IS NULL) AND ($2='' OR s.project=$2) + AND NOT COALESCE((SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id=b.branch_id AND c.field='excluded'),(SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id='' AND c.field='excluded'),false) +), members AS ( + SELECT b.* FROM eligible b WHERE NOT trashed OR NOT EXISTS(SELECT 1 FROM eligible v WHERE v.session_id=b.session_id AND NOT v.trashed) +), effective AS ( + SELECT DISTINCT ON(b.session_id,p.message_key) b.session_id,p.message_key,p.ordinal,p.note,p.created_at + FROM members b JOIN raw_pins p ON p.group_id=b.group_id AND (p.branch_id=b.branch_id OR p.branch_id='') + WHERE p.pinned AND (p.branch_id<>'' OR NOT EXISTS(SELECT 1 FROM raw_pins x WHERE x.group_id=b.group_id AND x.branch_id=b.branch_id AND x.message_key=p.message_key)) + ORDER BY b.session_id,p.message_key,(p.branch_id<>'') DESC,b.branch_id +)` + +// pinReferencePage limits effective references before fetching any transcript. +// Legacy and raw references share the same deterministic page, so unresolved +// pins compete by durable creation time, not by transient materialization IDs. +func (s *RawProjectionStore) pinReferencePage(ctx context.Context, id, project string) ([]db.PinnedMessage, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + rows, err := tx.QueryContext(ctx, rawEffectivePinCandidatesSQL+`, candidates AS ( + SELECT session_id,message_key,ordinal,note,created_at,0::bigint legacy_id FROM effective + UNION ALL + SELECT p.session_id,''::text,p.ordinal,p.note,p.created_at,p.id FROM pinned_messages p JOIN sessions s ON s.id=p.session_id + WHERE s.provenance_kind='legacy' AND ($1='' OR s.id=$1) AND ($1<>'' OR s.deleted_at IS NULL) AND ($2='' OR s.project=$2) + ), page AS MATERIALIZED ( + SELECT * FROM candidates ORDER BY created_at DESC,session_id,message_key,legacy_id DESC LIMIT CASE WHEN $1='' THEN 500 ELSE NULL END + ) + SELECT p.session_id,p.message_key,p.ordinal,p.note,p.created_at,COALESCE(pm.id,0),s.project,s.agent,COALESCE(s.display_name,s.session_name),s.first_message,m.content,m.role + FROM page p JOIN sessions s ON s.id=p.session_id + LEFT JOIN pinned_messages pm ON pm.session_id=p.session_id AND pm.ordinal=p.ordinal + LEFT JOIN messages m ON m.session_id=p.session_id AND m.ordinal=p.ordinal + ORDER BY p.created_at DESC,p.session_id,p.message_key,p.legacy_id DESC`, id, project) + if err != nil { + return nil, err + } + var pins []db.PinnedMessage + var rawIDs []string + for rows.Next() { + var p db.PinnedMessage + var created time.Time + if err = rows.Scan(&p.SessionID, &p.MessageKey, &p.Ordinal, &p.Note, &created, &p.ID, &p.SessionProject, &p.SessionAgent, &p.SessionDisplayName, &p.SessionFirstMessage, &p.Content, &p.Role); err != nil { + rows.Close() + return nil, err + } + p.CreatedAt = FormatISO8601(created) + p.MessageID = int64(p.Ordinal) + if p.MessageKey != "" { + rawIDs = append(rawIDs, p.SessionID) + p.Unresolved = true + } + pins = append(pins, p) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + slices.Sort(rawIDs) + rawIDs = slices.Compact(rawIDs) + if len(rawIDs) > 0 { + rows, err = tx.QueryContext(ctx, `SELECT session_id,payload FROM raw_content_revisions WHERE session_id=ANY($1)`, rawIDs) + if err != nil { + return nil, err + } + current := map[string]map[int]string{} + wanted := map[string]map[int]bool{} + for _, p := range pins { + if p.MessageKey != "" { + if wanted[p.SessionID] == nil { + wanted[p.SessionID] = map[int]bool{} + } + wanted[p.SessionID][p.Ordinal] = true + } + } + for rows.Next() { + var session string + var data []byte + if err = rows.Scan(&session, &data); err != nil { + rows.Close() + return nil, err + } + payload, e := decodeRawPayload(data) + if e != nil { + rows.Close() + return nil, e + } + keys := map[int]string{} + for _, m := range payload.Messages { + if !wanted[session][m.Ordinal] { + continue + } + key, e := rawMessageKey(m) + if e != nil { + rows.Close() + return nil, e + } + keys[m.Ordinal] = key + } + current[session] = keys + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + for i := range pins { + p := &pins[i] + if p.MessageKey != "" { + p.Unresolved = current[p.SessionID][p.Ordinal] != p.MessageKey + if p.Unresolved { + p.ID = 0 + p.MessageID = 0 + p.Content = nil + p.Role = nil + } + } + } + } + if err = tx.Commit(); err != nil { + return nil, err + } + return pins, nil +} diff --git a/internal/postgres/raw_projection_pin_removal_pgtest_test.go b/internal/postgres/raw_projection_pin_removal_pgtest_test.go new file mode 100644 index 0000000000..bb1977175b --- /dev/null +++ b/internal/postgres/raw_projection_pin_removal_pgtest_test.go @@ -0,0 +1,59 @@ +//go:build pgtest + +package postgres + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRawProjectionRemovesUnresolvedPinAcrossCurrentCohort(t *testing.T) { + f := newProjectionFixture(t) + a, ar := f.accept(t, "device-a", "old-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("original"))) + b, br := f.accept(t, "device-b", "old-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("original"))) + aliasA, aliasB := f.alias(t, a), f.alias(t, b) + require.NoError(t, f.sink.SetPin(t.Context(), "codex:portable", 0, true, "group note")) + refs, err := f.sink.PinReferences(t.Context(), aliasA) + require.NoError(t, err) + require.Len(t, refs, 1) + oldKey := refs[0].MessageKey + a, ar = f.accept(t, "device-a", "changed-a", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("replacement"))) + b, br = f.accept(t, "device-b", "changed-b", br.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("replacement"))) + refs, err = f.sink.PinReferences(t.Context(), aliasA) + require.NoError(t, err) + require.Len(t, refs, 1) + assert.False(t, refs[0].Resolved) + require.NoError(t, f.sink.RemovePin(t.Context(), aliasA, oldKey)) + for _, alias := range []string{aliasA, aliasB} { + refs, err = f.sink.PinReferences(t.Context(), alias) + require.NoError(t, err) + assert.Empty(t, refs) + } + // Both members keep the explicit unpin after a split, even when the original + // message becomes resolvable again on one branch. + b, br = f.accept(t, "device-b", "original-b", br.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("original"))) + refs, err = f.sink.PinReferences(t.Context(), aliasB) + require.NoError(t, err) + assert.Empty(t, refs) + require.NoError(t, f.sink.SetPin(t.Context(), aliasB, 0, true, "branch note")) + b, br = f.accept(t, "device-b", "replacement-b-again", br.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("replacement"))) + refs, err = f.sink.PinReferences(t.Context(), "codex:portable") + require.NoError(t, err) + require.Len(t, refs, 1) + assert.False(t, refs[0].Resolved) + require.NoError(t, f.sink.RemovePin(t.Context(), "codex:portable", oldKey)) + require.NoError(t, f.sink.RemovePin(t.Context(), "codex:portable", oldKey)) + b, _ = f.accept(t, "device-b", "original-b-after-base-clear", br.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("original"))) + refs, err = f.sink.PinReferences(t.Context(), aliasB) + require.NoError(t, err) + assert.Empty(t, refs, "base removal supersedes branch pin override") +} diff --git a/internal/postgres/raw_projection_pins.go b/internal/postgres/raw_projection_pins.go new file mode 100644 index 0000000000..d8ce3188be --- /dev/null +++ b/internal/postgres/raw_projection_pins.go @@ -0,0 +1,263 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "slices" + "time" +) + +// RawPinReference remains durable when no current message matches its key. +// Ordinal alone is never sufficient to resolve or reattach a pin. +type RawPinReference struct { + MessageKey string + Ordinal int + ContentRevision, Note string + Pinned, Resolved bool + CreatedAt time.Time +} + +func (s *RawProjectionStore) SetPin(ctx context.Context, alias string, ordinal int, pinned bool, note string) error { + _, err := s.SetPinReturningID(ctx, alias, ordinal, pinned, note) + return err +} + +// SetPinReturningID captures the response in the transaction that selects the +// cohort. A later publication may replace materialized rows, but cannot turn a +// committed pin into a failed response using an earlier physical identity. +func (s *RawProjectionStore) SetPinReturningID(ctx context.Context, alias string, ordinal int, pinned bool, note string) (int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + target, err := s.lockRawCurationTarget(ctx, tx, alias) + if err != nil { + return 0, err + } + payload, err := loadRawPayload(ctx, tx, target.SessionID) + if err != nil { + return 0, err + } + var key string + for _, m := range payload.Messages { + if m.Ordinal == ordinal { + key, err = rawMessageKey(m) + if err != nil { + return 0, err + } + break + } + } + if key == "" { + return 0, fmt.Errorf("raw pin message is absent") + } + if err = writeRawPin(ctx, tx, target, key, ordinal, target.ContentRevision, pinned, note); err != nil { + return 0, err + } + if err = s.publishRawCuration(ctx, tx, target.GroupID); err != nil { + return 0, err + } + var id int64 + if pinned { + if err = tx.QueryRowContext(ctx, `SELECT id FROM pinned_messages WHERE session_id=$1 AND ordinal=$2`, target.SessionID, ordinal).Scan(&id); err != nil { + return 0, err + } + } + if err = tx.Commit(); err != nil { + return 0, err + } + return id, nil +} + +// RemovePin removes a durable message identity even when no current ordinal +// resolves it. Group and current-cohort overrides follow the same rules as SetPin. +func (s *RawProjectionStore) RemovePin(ctx context.Context, alias, messageKey string) error { + if messageKey == "" { + return fmt.Errorf("raw pin message key is required") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + target, err := s.lockRawCurationTarget(ctx, tx, alias) + if err != nil { + return err + } + var ordinal int + var revision string + err = tx.QueryRowContext(ctx, `SELECT ordinal,content_revision FROM raw_pins WHERE group_id=$1 AND message_key=$2 ORDER BY branch_id LIMIT 1`, target.GroupID, messageKey).Scan(&ordinal, &revision) + if errors.Is(err, sql.ErrNoRows) { + return tx.Commit() + } + if err != nil { + return err + } + if err = writeRawPin(ctx, tx, target, messageKey, ordinal, revision, false, ""); err != nil { + return err + } + if err = s.publishRawCuration(ctx, tx, target.GroupID); err != nil { + return err + } + return tx.Commit() +} + +func writeRawPin(ctx context.Context, tx *sql.Tx, target RawIdentity, key string, ordinal int, revision string, pinned bool, note string) error { + var err error + if target.AnchorBranch == "" { + _, err = tx.ExecContext(ctx, `DELETE FROM raw_pins WHERE group_id=$1 AND message_key=$2 AND branch_id<>''`, target.GroupID, key) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_pins(group_id,branch_id,message_key,ordinal,content_revision,pinned,note) VALUES($1,'',$2,$3,$4,$5,$6) ON CONFLICT(group_id,branch_id,message_key) DO UPDATE SET ordinal=EXCLUDED.ordinal,content_revision=EXCLUDED.content_revision,pinned=EXCLUDED.pinned,note=EXCLUDED.note,created_at=CASE WHEN raw_pins.pinned THEN raw_pins.created_at ELSE clock_timestamp() END`, target.GroupID, key, ordinal, revision, pinned, note) + } else { + members, memberErr := rawCurationMembers(ctx, tx, target) + if memberErr != nil { + return memberErr + } + _, err = tx.ExecContext(ctx, `INSERT INTO raw_pins(group_id,branch_id,message_key,ordinal,content_revision,pinned,note) SELECT group_id,branch_id,$3,$4,$5,$6,$7 FROM raw_session_branches WHERE group_id=$1 AND session_id=$2 AND active AND branch_id=ANY($8) ON CONFLICT(group_id,branch_id,message_key) DO UPDATE SET ordinal=EXCLUDED.ordinal,content_revision=EXCLUDED.content_revision,pinned=EXCLUDED.pinned,note=EXCLUDED.note,created_at=CASE WHEN raw_pins.pinned THEN raw_pins.created_at ELSE clock_timestamp() END`, target.GroupID, target.SessionID, key, ordinal, revision, pinned, note, members) + } + if err != nil { + return err + } + return nil +} + +func (s *RawProjectionStore) PinReferences(ctx context.Context, alias string) ([]RawPinReference, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + target, err := resolveRawIdentity(ctx, tx, alias) + if err != nil { + return nil, err + } + if target.State != RawIdentityUnique { + return nil, &RawIdentityError{target} + } + branches, err := loadRawBranches(ctx, tx, "group_id", target.GroupID) + if err != nil { + return nil, err + } + overlays, err := loadRawOverlays(ctx, tx, target.GroupID) + if err != nil { + return nil, err + } + var members, visible []rawBranch + for _, b := range branches { + if b.Active && !overlays.boolean(b.ID, "excluded") && b.Session == target.SessionID { + members = append(members, b) + if !overlays.boolean(b.ID, "trashed") { + visible = append(visible, b) + } + } + } + if len(visible) == 0 { + visible = members + } + refs, err := rawEffectivePins(ctx, tx, target.GroupID, target.SessionID, visible) + if err != nil { + return nil, err + } + return refs, tx.Commit() +} +func rawEffectivePins(ctx context.Context, tx *sql.Tx, group, id string, members []rawBranch) ([]RawPinReference, error) { + rows, err := tx.QueryContext(ctx, `SELECT branch_id,message_key,ordinal,content_revision,pinned,note,created_at FROM raw_pins WHERE group_id=$1 ORDER BY branch_id,message_key`, group) + if err != nil { + return nil, err + } + overlays := map[string]map[string]RawPinReference{} + keys := map[string]bool{} + for rows.Next() { + var branch string + var p RawPinReference + if err = rows.Scan(&branch, &p.MessageKey, &p.Ordinal, &p.ContentRevision, &p.Pinned, &p.Note, &p.CreatedAt); err != nil { + rows.Close() + return nil, err + } + if overlays[branch] == nil { + overlays[branch] = map[string]RawPinReference{} + } + overlays[branch][p.MessageKey] = p + keys[p.MessageKey] = true + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + payload, err := loadRawPayload(ctx, tx, id) + if err != nil { + return nil, err + } + current := map[string]int{} + for _, m := range payload.Messages { + key, err := rawMessageKey(m) + if err != nil { + return nil, err + } + current[key] = m.Ordinal + } + return selectRawEffectivePins(overlays, current, members), nil +} +func selectRawEffectivePins(overlays map[string]map[string]RawPinReference, current map[string]int, members []rawBranch) []RawPinReference { + keys := map[string]bool{} + for _, pins := range overlays { + for key := range pins { + keys[key] = true + } + } + ordered := make([]string, 0, len(keys)) + for key := range keys { + ordered = append(ordered, key) + } + slices.Sort(ordered) + var refs []RawPinReference + for _, key := range ordered { + var selected *RawPinReference + explicit := false + for _, b := range members { + p, override := overlays[b.ID][key] + if !override { + p = overlays[""][key] + } + if !p.Pinned { + continue + } + if selected == nil || override && !explicit { + copy := p + selected = © + explicit = override + } + } + if selected != nil { + ordinal, ok := current[key] + selected.Resolved = ok && ordinal == selected.Ordinal + refs = append(refs, *selected) + } + } + return refs +} +func materializeRawPins(ctx context.Context, tx *sql.Tx, group, id string, members []rawBranch) error { + refs, err := rawEffectivePins(ctx, tx, group, id, members) + if err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM pinned_messages WHERE session_id=$1`, id); err != nil { + return err + } + for _, p := range refs { + if !p.Resolved { + continue + } + _, err = tx.ExecContext(ctx, `INSERT INTO pinned_messages(session_id,message_id,ordinal,source_uuid,note,created_at) SELECT session_id,ordinal,ordinal,source_uuid,$3,$4 FROM messages WHERE session_id=$1 AND ordinal=$2`, id, p.Ordinal, p.Note, p.CreatedAt) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/postgres/raw_projection_provenance_pgtest_test.go b/internal/postgres/raw_projection_provenance_pgtest_test.go new file mode 100644 index 0000000000..fdc94e79e3 --- /dev/null +++ b/internal/postgres/raw_projection_provenance_pgtest_test.go @@ -0,0 +1,311 @@ +//go:build pgtest + +package postgres + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +func TestRawProjectionPreservesAbsentProviderTitleFromSameSource(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "named", "") + named := projectionOutcome("first") + named.Outcome.Results[0].Result.Session.SessionName = "Provider title" + named.Outcome.Results[0].Result.Session.SessionNamePresent = true + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, named)) + next, receipt := f.accept(t, "device-a", "unnamed", accepted.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("changed content"))) + identity, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + var title *string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT session_name FROM sessions WHERE id=$1`, identity.SessionID).Scan(&title)) + require.NotNil(t, title) + assert.Equal(t, "Provider title", *title) + cleared, _ := f.accept(t, "device-a", "cleared", receipt.Receipt) + explicit := projectionOutcome("changed content") + explicit.Outcome.Results[0].Result.Session.SessionNamePresent = true + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, cleared), cleared, explicit)) + identity, err = f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT session_name FROM sessions WHERE id=$1`, identity.SessionID).Scan(&title)) + assert.Nil(t, title) +} + +func TestRawProjectionEqualSourcesNoOpAndRemoval(t *testing.T) { + f := newProjectionFixture(t) + a, ar := f.accept(t, "device-a", "equal-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, projectionOutcome("equal"))) + first, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + a2, ar2 := f.accept(t, "device-a", "equal-a-retry", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a2), a2, projectionOutcome("equal"))) + retry, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, first.CorpusRevision, retry.CorpusRevision) + assert.Equal(t, first.IdentityRevision, retry.IdentityRevision) + b, _ := f.accept(t, "device-b", "equal-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, projectionOutcome("equal"))) + var count int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM sessions`).Scan(&count)) + assert.Equal(t, 1, count) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM session_sources`).Scan(&count)) + assert.Equal(t, 2, count) + oldAlias := f.alias(t, a) + require.NoError(t, f.sink.SetCuration(t.Context(), oldAlias, "display_name", "retained name")) + removed, rr := f.accept(t, "device-a", "removed-a", ar2.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, removed), removed, rawderive.ParsedManifest{Outcome: parser.ParseOutcome{ResultSetComplete: true}})) + gone, err := f.sink.Resolve(t.Context(), oldAlias) + require.NoError(t, err) + assert.Equal(t, RawIdentityGone, gone.State) + surviving, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, surviving.State) + assert.Equal(t, first.SessionID, surviving.SessionID) + returned, _ := f.accept(t, "device-a", "returned-a", rr.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, returned), returned, projectionOutcome("returned"))) + again, err := f.sink.Resolve(t.Context(), oldAlias) + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, again.State) + var name string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT display_name FROM sessions WHERE id=$1`, again.SessionID).Scan(&name)) + assert.Equal(t, "retained name", name) + private, err := f.sink.Resolve(t.Context(), again.SessionID) + require.NoError(t, err) + assert.Equal(t, RawIdentityGone, private.State) +} + +func TestRawProjectionPartialMembersKeepPriorProof(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "multi-a", "") + outcome := projectionOutcome("first") + second := projectionOutcome("second").Outcome.Results[0] + second.Result.Session.ID = "codex:second" + second.Result.Session.SourceSessionID = "second" + outcome.Outcome.Results = append(outcome.Outcome.Results, second) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, outcome)) + previous, err := f.sink.Resolve(t.Context(), "codex:second") + require.NoError(t, err) + next, _ := f.accept(t, "device-a", "multi-b", accepted.Receipt) + lease := f.lease(t, next) + partial := projectionOutcome("updated first") + partial.Outcome.ResultSetComplete = false + partial.Outcome.SourceErrors = []parser.SourceError{{SessionID: "codex:second", Err: errors.New("synthetic parse failure"), Retryable: true}} + require.ErrorIs(t, f.sink.Project(t.Context(), lease, next, partial), rawderive.ProjectionRetrying) + retained, err := f.sink.Resolve(t.Context(), "codex:second") + require.NoError(t, err) + assert.Equal(t, previous.SessionID, retained.SessionID) + var proof, state string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT manifest_id FROM session_sources WHERE session_id=$1`, retained.SessionID).Scan(&proof)) + assert.Equal(t, m.ManifestID, proof) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT state FROM raw_ingest_jobs WHERE id=$1`, lease.ID).Scan(&state)) + assert.Equal(t, "retrying", state) +} +func TestRawProjectionRetainsProviderHistoryAndEveryAcceptedContributor(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "history-a", "", parser.AgentRooCode) + outcome := projectionOutcome("retained history") + outcome.Outcome.Results[0].Result.Session.Agent = parser.AgentRooCode + outcome.Outcome.Results[0].Result.Session.ID = "roocode:portable" + outcome.Outcome.Results[0].Result.Session.File = parser.FileInfo{Path: "legacy/source.json", Size: 123, Mtime: 456, Hash: "prior-hash"} + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, outcome)) + first, err := f.sink.Resolve(t.Context(), "roocode:portable") + require.NoError(t, err) + next, _ := f.accept(t, "device-a", "history-b", accepted.Receipt, parser.AgentRooCode) + outcome.Outcome.Results[0].Result.Messages = nil + outcome.Outcome.Results[0].Result.UsageEvents = nil + outcome.Outcome.Results[0].Result.Session.File = parser.FileInfo{Path: "legacy/source.json", Size: 41, Mtime: 789, Hash: "empty-current-hash"} + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, outcome)) + retained, err := f.sink.Resolve(t.Context(), "roocode:portable") + require.NoError(t, err) + assert.Equal(t, first.SessionID, retained.SessionID) + messages, err := (&Store{pg: f.runtime}).GetMessages(t.Context(), retained.SessionID, 0, 10, true) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "retained history", messages[0].Content) + usage, err := (&Store{pg: f.runtime}).GetSessionUsageRows(t.Context(), []string{retained.SessionID}) + require.NoError(t, err) + require.Len(t, usage.Rows, 1) + assert.Equal(t, 7, usage.Rows[0].OutputTokens) + rows, err := f.runtime.QueryContext(t.Context(), `SELECT c.manifest_id,c.projection_generation,c.processing_version,c.prior_contributed,c.payload,s.device_id,s.provider FROM raw_source_contributions c JOIN raw_session_branches b ON b.branch_id=c.branch_id JOIN raw_source_projections s ON s.source_id=b.source_id ORDER BY c.projection_generation`) + require.NoError(t, err) + defer rows.Close() + for i, expected := range []struct { + manifest, hash string + size, mtime int64 + contributed bool + }{ + {m.ManifestID, "prior-hash", 123, 456, false}, + {next.ManifestID, "empty-current-hash", 41, 789, true}, + } { + require.True(t, rows.Next()) + var manifest, version, device, provider string + var generation int64 + var contributed bool + var payload []byte + require.NoError(t, rows.Scan(&manifest, &generation, &version, &contributed, &payload, &device, &provider)) + assert.Equal(t, expected.manifest, manifest) + assert.Equal(t, int64(i+1), generation) + assert.Equal(t, "parser-1", version) + assert.Equal(t, expected.contributed, contributed) + assert.Equal(t, "device-a", device) + assert.Equal(t, string(parser.AgentRooCode), provider) + metadata, err := decodeRawPayload(payload) + require.NoError(t, err) + assert.Equal(t, new("legacy/source.json"), metadata.Session.FilePath) + assert.Equal(t, new(expected.hash), metadata.Session.FileHash) + assert.Equal(t, new(expected.size), metadata.Session.FileSize) + assert.Equal(t, new(expected.mtime), metadata.Session.FileMtime) + assert.Empty(t, metadata.Messages, "contribution metadata must not duplicate the shared transcript") + } + assert.False(t, rows.Next()) + require.NoError(t, rows.Err()) +} + +func TestRawProjectionUsageOnlyMemberAndDerivedFindings(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "complete-graph", "") + outcome := projectionOutcome("inspect") + outcome.Outcome.Results[0].Result.Messages[1].ToolCalls[0].InputJSON = `{"token":"AKIA7QHWN2DKR4FYPLJM"}` + outcome.Outcome.Results[0].Result.Messages[1].ToolCalls[0].ResultEvents = []parser.ParsedToolResultEvent{{ToolUseID: "call-1", Source: "tool", Status: "completed", Content: "synthetic result"}} + zero := projectionOutcome("unused").Outcome.Results[0] + zero.Result.Session.ID = "codex:usage-only" + zero.Result.Session.SourceSessionID = "usage-only" + zero.Result.Messages = nil + outcome.Outcome.Results = append(outcome.Outcome.Results, zero) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, outcome)) + graph, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + store := &Store{pg: f.runtime} + messages, err := store.GetMessages(t.Context(), graph.SessionID, 0, 10, true) + require.NoError(t, err) + require.Len(t, messages, 2) + require.Len(t, messages[1].ToolCalls, 1) + require.Len(t, messages[1].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, "synthetic result", messages[1].ToolCalls[0].ResultEvents[0].Content) + findings, err := store.ListSecretFindings(t.Context(), db.SecretFindingFilter{}) + require.NoError(t, err) + require.NotEmpty(t, findings.Findings) + assert.Equal(t, "tool_input", findings.Findings[0].LocationKind) + assert.Equal(t, 1, findings.Findings[0].MessageOrdinal) + session, err := store.GetSession(t.Context(), graph.SessionID) + require.NoError(t, err) + require.NotNil(t, session) + assert.Positive(t, session.SecretLeakCount) + usageOnly, err := f.sink.Resolve(t.Context(), "codex:usage-only") + require.NoError(t, err) + rows, err := store.GetSessionUsageRows(t.Context(), []string{usageOnly.SessionID}) + require.NoError(t, err) + require.Len(t, rows.Rows, 1) + assert.Equal(t, 7, rows.Rows[0].OutputTokens) +} + +func TestRawProjectionKeepsLegacyProofSeparate(t *testing.T) { + f := newProjectionFixture(t) + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent,first_message) VALUES('codex:portable','legacy','archive','codex','legacy preserved')`) + require.NoError(t, err) + m, _ := f.accept(t, "device-a", "legacy-separate", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("raw"))) + var kind, message string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT provenance_kind,first_message FROM sessions WHERE id='codex:portable'`).Scan(&kind, &message)) + assert.Equal(t, "legacy", kind) + assert.Equal(t, "legacy preserved", message) + r, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT provenance_kind FROM sessions WHERE id=$1`, r.SessionID).Scan(&kind)) + assert.Equal(t, "raw", kind) +} + +func TestRawProjectionTombstoneHidesEveryPhysicalInventoryAndReactivationRestoresCuration(t *testing.T) { + f := newProjectionFixture(t) + m, accepted := f.accept(t, "device-a", "visible", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("needle"))) + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "starred", true)) + require.NoError(t, f.sink.SetPin(t.Context(), "codex:portable", 0, true, "retained")) + store := &Store{pg: f.runtime} + before, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + tomb := rawsync.Manifest{SchemaVersion: rawsync.ManifestSchemaVersion, Provider: m.Manifest.Provider, ConfiguredRootID: m.Manifest.ConfiguredRootID, SourceKey: m.Manifest.SourceKey, ExpectedParentReceipt: accepted.Receipt, CaptureID: "tombstone", CapturedAt: rawIngestCapturedAt(), Kind: rawsync.ManifestTombstone} + canonical, err := rawsync.ValidateAndCanonicalize(m.Identity, tomb, rawsync.DefaultManifestLimits()) + require.NoError(t, err) + removed, err := f.custody.CommitManifest(t.Context(), m.Identity, tomb) + require.NoError(t, err) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, canonical), canonical, rawderive.ParsedManifest{Tombstone: true})) + page, err := store.ListSessions(t.Context(), db.SessionFilter{}) + require.NoError(t, err) + assert.Empty(t, page.Sessions) + trash, err := store.ListTrashedSessions(t.Context()) + require.NoError(t, err) + assert.Empty(t, trash) + stars, err := store.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Empty(t, stars) + matches, err := store.Search(t.Context(), db.SearchFilter{Query: "needle"}) + require.NoError(t, err) + assert.Empty(t, matches.Results) + next, _ := f.accept(t, "device-a", "reactivated", removed.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, projectionOutcome("needle"))) + after, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, before.SessionID, after.SessionID) + stars, err = store.ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{after.SessionID}, stars) + pins, err := store.ListPinnedMessages(t.Context(), after.SessionID, "") + require.NoError(t, err) + require.Len(t, pins, 1) +} + +func TestRawProjectionProviderFallbackAliasSurvivesConflict(t *testing.T) { + f := newProjectionFixture(t) + for _, device := range []string{"device-a", "device-b"} { + m, _ := f.accept(t, device, "vibe-"+device, "", parser.AgentVibe) + out := projectionOutcome(device) + out.Outcome.Results[0].Result.Session.Agent = parser.AgentVibe + out.Outcome.Results[0].Result.Session.ID = "vibe:portable" + out.Outcome.Results[0].Result.Session.File.Path = "logs/session_fallback/messages.jsonl" + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + } + alias, err := f.sink.Resolve(t.Context(), "vibe:session_fallback") + require.NoError(t, err) + assert.Equal(t, RawIdentityAmbiguous, alias.State) + require.Len(t, alias.Variants, 2) + for _, variant := range alias.Variants { + resolved, err := f.sink.Resolve(t.Context(), variant) + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, resolved.State) + } +} + +func TestRawProjectionNonportableIdentityHasExplicitAliasConflict(t *testing.T) { + f := newProjectionFixture(t) + a, ar := f.accept(t, "device-a", "opaque-a", "") + out := projectionOutcome("equal text") + out.Outcome.Results[0].Result.Session.SourceSessionID = "" + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, out)) + b, _ := f.accept(t, "device-b", "opaque-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, out)) + r, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityAmbiguous, r.State) + require.Len(t, r.Variants, 2) + for _, alias := range r.Variants { + variant, err := f.sink.Resolve(t.Context(), alias) + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, variant.State) + assert.Equal(t, alias, variant.PublicID) + } + next, _ := f.accept(t, "device-a", "opaque-removed", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, next), next, rawderive.ParsedManifest{Outcome: parser.ParseOutcome{ResultSetComplete: true}})) + remaining, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, remaining.State) + assert.Equal(t, "codex:portable", remaining.PublicID) +} diff --git a/internal/postgres/raw_projection_rows.go b/internal/postgres/raw_projection_rows.go new file mode 100644 index 0000000000..af833dfa89 --- /dev/null +++ b/internal/postgres/raw_projection_rows.go @@ -0,0 +1,149 @@ +package postgres + +import ( + "context" + "database/sql" + "reflect" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" +) + +const ( + rawSourceProofDeleteSQL = `DELETE FROM session_sources WHERE source_id=$1` + rawSourceProofAttachSQL = `UPDATE session_sources SET physical_session_id=$1 WHERE session_id=$1` + rawSourceProofDetachSQL = `UPDATE session_sources SET physical_session_id=NULL WHERE physical_session_id=$1` + rawGroupContentsSQL = `SELECT id,raw_content_revision FROM sessions WHERE raw_group_id=$1 AND raw_group_id<>'' UNION SELECT session_id,content_revision FROM raw_session_branches WHERE group_id=$1 AND active ORDER BY 1` +) + +func (s *RawProjectionStore) materializeGroup(ctx context.Context, tx *sql.Tx, group string, corpus int64) error { + branches, err := loadRawBranches(ctx, tx, "group_id", group) + if err != nil { + return err + } + overlays, err := loadRawOverlays(ctx, tx, group) + if err != nil { + return err + } + cohorts := map[string][]rawBranch{} + for _, b := range branches { + if b.Active && !overlays.boolean(b.ID, "excluded") { + cohorts[b.Session] = append(cohorts[b.Session], b) + } + } + rows, err := tx.QueryContext(ctx, rawGroupContentsSQL, group) + if err != nil { + return err + } + type content struct{ id, revision string } + var contents []content + for rows.Next() { + var c content + if err = rows.Scan(&c.id, &c.revision); err != nil { + rows.Close() + return err + } + contents = append(contents, c) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, c := range contents { + exists, err := rawPhysicalExists(ctx, tx, c.id) + if err != nil { + return err + } + members := cohorts[c.id] + action := "reconcile" + if len(members) == 0 { + if !exists { + continue + } + if _, err = tx.ExecContext(ctx, rawSourceProofDetachSQL, c.id); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM sessions WHERE id=$1`, c.id); err != nil { + return err + } + action = "remove" + } else { + if !exists { + payload, err := loadRawPayload(ctx, tx, c.id) + if err != nil { + return err + } + if err = s.writePayload(ctx, tx, c.id, group, c.revision, payload); err != nil { + return err + } + } + if _, err = tx.ExecContext(ctx, rawSourceProofAttachSQL, c.id); err != nil { + return err + } + if err = s.materializeCuration(ctx, tx, group, c.id, members); err != nil { + return err + } + } + if !exists || action == "remove" { + _, err = tx.ExecContext(ctx, `INSERT INTO raw_embedding_outbox(session_id,corpus_revision,content_revision,action,selection_revision) SELECT $1,$2,$3,$4,selection_revision FROM raw_corpus_state WHERE singleton=1 ON CONFLICT DO NOTHING`, c.id, corpus, c.revision, action) + if err != nil { + return err + } + } + } + return nil +} +func (s *RawProjectionStore) writePayload(ctx context.Context, tx *sql.Tx, id, group, revision string, p ingest.PreparedSession) error { + p.Session.ID = id + p.Session.FilePath = nil + p.Session.Machine = "hosted" + p.Session.CreatedAt = time.Now().UTC().Format(time.RFC3339Nano) + p.Session.TranscriptRevision = &revision + p.Session.DataVersion = db.CurrentDataVersion() + // Full shared preparation computes signals independently of Session. Copy + // matching scalar fields, then the quality scalar fields expected by the + // existing SQL kernel. This does not encode public JSON or omit hidden fields. + copyRawSignalFields(&p.Session, p.Signals) + for i := range p.UsageEvents { + p.UsageEvents[i].SessionID = id + } + for i := range p.Findings { + p.Findings[i].SessionID = id + } + if err := writePGSession(ctx, tx, p.Session, "raw-projection:"+group, nil, pgSessionWriteOptions{Machine: "hosted", UsageOnly: s.options.Content.ArchiveContent.UsageOnly(), SkipAliases: true}); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE sessions SET provenance_kind='raw',raw_group_id=$2,raw_content_revision=$3 WHERE id=$1`, id, group, revision); err != nil { + return err + } + if err := bulkInsertMessages(ctx, tx, id, p.Messages); err != nil { + return err + } + if err := bulkInsertToolCalls(ctx, tx, id, p.Messages); err != nil { + return err + } + if err := bulkInsertToolResultEvents(ctx, tx, id, p.Messages); err != nil { + return err + } + if err := bulkInsertUsageEvents(ctx, tx, p.UsageEvents); err != nil { + return err + } + return bulkInsertSecretFindings(ctx, tx, id, p.Findings) +} +func copyRawSignalFields(session *db.Session, signals db.SessionSignalUpdate) { + target := reflect.ValueOf(session).Elem() + for _, source := range []reflect.Value{reflect.ValueOf(signals), reflect.ValueOf(signals.QualitySignals)} { + for i := 0; i < source.NumField(); i++ { + name := source.Type().Field(i).Name + if name == "Version" { + name = "QualitySignalVersion" + } + field := target.FieldByName(name) + if field.IsValid() && field.CanSet() && field.Type() == source.Field(i).Type() { + field.Set(source.Field(i)) + } + } + } +} diff --git a/internal/postgres/raw_projection_schema.go b/internal/postgres/raw_projection_schema.go new file mode 100644 index 0000000000..a75c92bb12 --- /dev/null +++ b/internal/postgres/raw_projection_schema.go @@ -0,0 +1,177 @@ +package postgres + +import ( + "context" + "database/sql" +) + +// Projection metadata is durable independently of disposable physical payloads. +// Ordinary schema setup never creates these hosted-only relations. +const rawProjectionDDL = ` +CREATE TABLE IF NOT EXISTS raw_projection_rollouts ( + run_id TEXT PRIMARY KEY, processing_version TEXT NOT NULL, + device_id TEXT NOT NULL DEFAULT '', provider TEXT NOT NULL DEFAULT '', + configured_root_id TEXT NOT NULL DEFAULT '', source_key_sha256 TEXT NOT NULL DEFAULT '', + complete BOOLEAN NOT NULL DEFAULT false +); + +CREATE TABLE IF NOT EXISTS raw_source_projections ( + source_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, provider TEXT NOT NULL, + configured_root_id TEXT NOT NULL, source_key_sha256 TEXT NOT NULL, + selected_manifest_id TEXT NOT NULL, processing_version TEXT NOT NULL, + projection_generation BIGINT NOT NULL CHECK(projection_generation>0), + selected_job_id BIGINT NOT NULL, successful_manifest_id TEXT, last_attempt_manifest_id TEXT, + membership_complete BOOLEAN NOT NULL DEFAULT false, diagnostics TEXT NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS raw_projection_generations ( + source_id TEXT NOT NULL, generation BIGINT NOT NULL, manifest_id TEXT NOT NULL, + processing_version TEXT NOT NULL, PRIMARY KEY(source_id,generation) +); +CREATE TABLE IF NOT EXISTS raw_session_groups ( + group_id TEXT PRIMARY KEY, provider TEXT NOT NULL, logical_key TEXT NOT NULL, + base_alias TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS raw_content_revisions ( + session_id TEXT PRIMARY KEY, group_id TEXT NOT NULL, content_revision TEXT NOT NULL, + payload BYTEA NOT NULL, recency_state JSONB NOT NULL DEFAULT '{}', UNIQUE(group_id,content_revision) +); +CREATE TABLE IF NOT EXISTS raw_session_branches ( + branch_id TEXT PRIMARY KEY, source_id TEXT NOT NULL, group_id TEXT NOT NULL, + member_id TEXT NOT NULL, session_id TEXT NOT NULL, content_revision TEXT NOT NULL, + manifest_id TEXT NOT NULL, processing_version TEXT NOT NULL, + projection_generation BIGINT NOT NULL, active BOOLEAN NOT NULL, + prior_payload BYTEA NOT NULL, UNIQUE(source_id,group_id) +); +CREATE INDEX IF NOT EXISTS raw_branches_group ON raw_session_branches(group_id,active,session_id); +CREATE INDEX IF NOT EXISTS raw_branches_source ON raw_session_branches(source_id); +CREATE TABLE IF NOT EXISTS session_sources ( + branch_id TEXT PRIMARY KEY, group_id TEXT NOT NULL, source_id TEXT NOT NULL, + session_id TEXT NOT NULL, physical_session_id TEXT, manifest_id TEXT NOT NULL, content_revision TEXT NOT NULL, + processing_version TEXT NOT NULL, projection_generation BIGINT NOT NULL +); +CREATE TABLE IF NOT EXISTS raw_source_contributions ( + branch_id TEXT NOT NULL, manifest_id TEXT NOT NULL, projection_generation BIGINT NOT NULL, processing_version TEXT NOT NULL, prior_contributed BOOLEAN NOT NULL, payload BYTEA NOT NULL, + PRIMARY KEY(branch_id,manifest_id,projection_generation) +); +CREATE TABLE IF NOT EXISTS raw_session_public_aliases ( + alias_id TEXT NOT NULL, group_id TEXT NOT NULL, anchor_branch TEXT NOT NULL DEFAULT '', + PRIMARY KEY(alias_id,group_id) +); +CREATE TABLE IF NOT EXISTS raw_curation ( + group_id TEXT NOT NULL, branch_id TEXT NOT NULL DEFAULT '', field TEXT NOT NULL, + value JSONB NOT NULL, PRIMARY KEY(group_id,branch_id,field) +); +CREATE TABLE IF NOT EXISTS raw_pins ( + group_id TEXT NOT NULL, branch_id TEXT NOT NULL DEFAULT '', message_key TEXT NOT NULL, + ordinal INTEGER NOT NULL, content_revision TEXT NOT NULL, pinned BOOLEAN NOT NULL, + note TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), PRIMARY KEY(group_id,branch_id,message_key) +); +CREATE TABLE IF NOT EXISTS raw_corpus_state ( + singleton SMALLINT PRIMARY KEY CHECK(singleton=1), corpus_revision BIGINT NOT NULL DEFAULT 0, + identity_revision BIGINT NOT NULL DEFAULT 0, selection_revision BIGINT NOT NULL DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS raw_embedding_outbox ( + session_id TEXT NOT NULL, selection_revision BIGINT NOT NULL, corpus_revision BIGINT NOT NULL, content_revision TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('reconcile','remove')), + PRIMARY KEY(session_id,selection_revision,corpus_revision) +); +` + +const rawProjectionIndexesDDL = ` +CREATE INDEX IF NOT EXISTS raw_selected_job ON raw_source_projections(tenant_id,selected_job_id); +CREATE INDEX IF NOT EXISTS raw_pending_signals ON sessions(tenant_id,ended_at,id) WHERE provenance_kind='raw' AND signals_pending_since IS NOT NULL; +CREATE INDEX IF NOT EXISTS raw_hosted_jobs_due ON raw_ingest_jobs(tenant_id,processing_version,available_at,id) WHERE stage='parse' AND projection_selected AND projection_generation>0 AND state IN ('ready','retrying'); +CREATE INDEX IF NOT EXISTS raw_hosted_jobs_expired ON raw_ingest_jobs(tenant_id,processing_version,lease_expires_at,id) WHERE stage='parse' AND projection_selected AND projection_generation>0 AND state='leased'; + +CREATE INDEX IF NOT EXISTS raw_branches_session ON raw_session_branches(tenant_id,session_id,active); +CREATE INDEX IF NOT EXISTS raw_links_target ON raw_session_links(tenant_id,target_alias); +CREATE INDEX IF NOT EXISTS raw_alias_group ON raw_session_public_aliases(tenant_id,group_id); +CREATE INDEX IF NOT EXISTS raw_session_sources_source ON session_sources(tenant_id,source_id); +CREATE INDEX IF NOT EXISTS raw_session_sources_content ON session_sources(tenant_id,session_id); +CREATE INDEX IF NOT EXISTS raw_session_sources_physical ON session_sources(tenant_id,physical_session_id); +` + +var rawProjectionTables = []HostedTable{ + {Name: "raw_projection_rollouts", Key: []string{"run_id"}}, + {Name: "raw_session_links", Key: []string{"branch_id", "kind", "ordinal", "call_index", "event_index"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"branch_id"}, Table: "raw_session_branches", References: []string{"branch_id"}, Delete: "RESTRICT"}}}, + {Name: "raw_source_projections", Key: []string{"source_id"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"device_id", "provider", "configured_root_id", "source_key_sha256"}, Table: "raw_source_heads", References: []string{"device_id", "provider", "configured_root_id", "source_key_sha256"}, Delete: "RESTRICT"}, + {Columns: []string{"selected_manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + {Columns: []string{"selected_job_id"}, Table: "raw_ingest_jobs", References: []string{"id"}, Delete: "RESTRICT"}, + {Columns: []string{"successful_manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + {Columns: []string{"last_attempt_manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_projection_generations", Key: []string{"source_id", "generation"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"source_id"}, Table: "raw_source_projections", References: []string{"source_id"}, Delete: "RESTRICT"}, + {Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_session_groups", Key: []string{"group_id"}}, + {Name: "raw_content_revisions", Key: []string{"session_id"}, ForeignKeys: rawGroupFK()}, + {Name: "raw_session_branches", Key: []string{"branch_id"}, ForeignKeys: append(rawGroupFK(), + HostedForeignKey{Columns: []string{"source_id"}, Table: "raw_source_projections", References: []string{"source_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"session_id"}, Table: "raw_content_revisions", References: []string{"session_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"})}, + {Name: "session_sources", Key: []string{"branch_id"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"physical_session_id"}, Table: "sessions", References: []string{"id"}, Delete: "RESTRICT"}, + {Columns: []string{"session_id"}, Table: "raw_content_revisions", References: []string{"session_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"branch_id"}, Table: "raw_session_branches", References: []string{"branch_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"group_id"}, Table: "raw_session_groups", References: []string{"group_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"source_id"}, Table: "raw_source_projections", References: []string{"source_id"}, Delete: "RESTRICT"}, + HostedForeignKey{Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}}}, + {Name: "raw_source_contributions", Key: []string{"branch_id", "manifest_id", "projection_generation"}, ForeignKeys: []HostedForeignKey{ + {Columns: []string{"branch_id"}, Table: "raw_session_branches", References: []string{"branch_id"}, Delete: "RESTRICT"}, + {Columns: []string{"manifest_id"}, Table: "raw_manifests", References: []string{"manifest_id"}, Delete: "RESTRICT"}, + }}, + {Name: "raw_session_public_aliases", Key: []string{"alias_id", "group_id"}, ForeignKeys: rawGroupFK()}, + {Name: "raw_curation", Key: []string{"group_id", "branch_id", "field"}, ForeignKeys: rawGroupFK()}, + {Name: "raw_pins", Key: []string{"group_id", "branch_id", "message_key"}, ForeignKeys: rawGroupFK()}, + {Name: "raw_corpus_state", Key: []string{"singleton"}}, + {Name: "raw_embedding_outbox", Key: []string{"session_id", "selection_revision", "corpus_revision"}, ForeignKeys: []HostedForeignKey{{Columns: []string{"session_id"}, Table: "raw_content_revisions", References: []string{"session_id"}, Delete: "RESTRICT"}}}, +} + +func rawGroupFK() []HostedForeignKey { + return []HostedForeignKey{{Columns: []string{"group_id"}, Table: "raw_session_groups", References: []string{"group_id"}, Delete: "RESTRICT"}} +} + +// installRawProjectionUpgrade adds only missing projection tables to an already +// bound tenant schema. The complete catalog check that follows still rejects +// tampered protections or unknown tables; failures roll back the entire upgrade. +func installRawProjectionUpgrade(ctx context.Context, tx *sql.Tx, schema, tenant string) error { + var missing []HostedTable + for _, table := range rawProjectionTables { + var exists bool + if err := tx.QueryRowContext(ctx, `SELECT to_regclass(format('%I.%I',$1::text,$2::text)) IS NOT NULL`, schema, table.Name).Scan(&exists); err != nil { + return err + } + if !exists { + missing = append(missing, table) + } + } + if len(missing) > 0 { + if _, err := tx.ExecContext(ctx, rawProjectionDDL+rawLinksDDL); err != nil { + return err + } + if err := InstallHostedTables(ctx, tx, schema, tenant, missing); err != nil { + return err + } + } + // Job generation is a shared raw-schema migration; projection does not change + // custody data or assign a generation to legacy jobs implicitly. + if err := ensureRawProjectionJobColumns(ctx, tx); err != nil { + return err + } + migrations := append(schemaColumnMigrations(), columnMigration{"raw_pins", "created_at", `created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()`, "adding durable raw pin creation time"}, columnMigration{"raw_content_revisions", "recency_state", `recency_state JSONB NOT NULL DEFAULT '{}'`, "adding raw content recency state"}) + columns, err := loadExistingColumns(ctx, tx, migrations) + if err != nil { + return err + } + _, err = ensureColumns(ctx, tx, columns, migrations) + if err != nil { + return err + } + if err := createPartialIndexesPG(ctx, tx); err != nil { + return err + } + _, err = tx.ExecContext(ctx, rawProjectionIndexesDDL) + return err +} diff --git a/internal/postgres/raw_projection_signals.go b/internal/postgres/raw_projection_signals.go new file mode 100644 index 0000000000..134d687478 --- /dev/null +++ b/internal/postgres/raw_projection_signals.go @@ -0,0 +1,64 @@ +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "reflect" + + "go.kenn.io/agentsview/internal/db" +) + +// Only these fields depend on observation time. Their small mutable snapshot +// belongs to a content revision but never participates in its immutable hash. +type rawRecencyState struct { + Outcome, OutcomeConfidence string + SignalsPendingSince *string + HealthScore *int + HealthGrade *string +} + +func decodeRawRecency(data []byte) (rawRecencyState, error) { + var state rawRecencyState + err := json.Unmarshal(data, &state) + return state, err +} +func (r rawRecencyState) apply(update *db.SessionSignalUpdate) { + update.Outcome, update.OutcomeConfidence = r.Outcome, r.OutcomeConfidence + update.SignalsPendingSince = r.SignalsPendingSince + update.HealthScore, update.HealthGrade = r.HealthScore, r.HealthGrade +} + +// publishRawRecency runs under the logical group lock. It persists settling for +// later reactivation and reports only a change to a currently materialized row. +// Equal recent observations retain the original pending timestamp. +func publishRawRecency(ctx context.Context, tx *sql.Tx, id string, update db.SessionSignalUpdate) (bool, error) { + next := rawRecencyState{update.Outcome, update.OutcomeConfidence, update.SignalsPendingSince, update.HealthScore, update.HealthGrade} + var data []byte + if err := tx.QueryRowContext(ctx, `SELECT recency_state FROM raw_content_revisions WHERE session_id=$1`, id).Scan(&data); err != nil { + return false, err + } + prior, err := decodeRawRecency(data) + if err != nil { + return false, err + } + if prior.SignalsPendingSince != nil && next.SignalsPendingSince != nil { + next.SignalsPendingSince = prior.SignalsPendingSince + } + if reflect.DeepEqual(prior, next) { + return false, nil + } + data, err = json.Marshal(next) + if err != nil { + return false, err + } + if _, err = tx.ExecContext(ctx, `UPDATE raw_content_revisions SET recency_state=$2 WHERE session_id=$1`, id, data); err != nil { + return false, err + } + result, err := tx.ExecContext(ctx, `UPDATE sessions SET outcome=$2,outcome_confidence=$3,signals_pending_since=$4,health_score=$5,health_grade=$6 WHERE id=$1 AND (outcome,outcome_confidence,signals_pending_since,health_score,health_grade) IS DISTINCT FROM ($2::text,$3::text,$4::text,$5::int,$6::text)`, id, next.Outcome, next.OutcomeConfidence, next.SignalsPendingSince, next.HealthScore, next.HealthGrade) + if err != nil { + return false, err + } + count, err := result.RowsAffected() + return count > 0, err +} diff --git a/internal/postgres/raw_projection_signals_pgtest_test.go b/internal/postgres/raw_projection_signals_pgtest_test.go new file mode 100644 index 0000000000..36560ba04a --- /dev/null +++ b/internal/postgres/raw_projection_signals_pgtest_test.go @@ -0,0 +1,78 @@ +//go:build pgtest + +package postgres + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" +) + +func TestRawProjectionEqualContentSettlesSignalsWithoutNewIdentity(t *testing.T) { + f := newProjectionFixture(t) + ended := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) + now := ended.Add(time.Minute) + f.sink.options.Now = func() time.Time { return now } + outcome := projectionOutcome("Implement the requested change") + outcome.Outcome.Results[0].Result.Session.EndedAt = ended + outcome.Outcome.Results[0].Result.Messages = append(outcome.Outcome.Results[0].Result.Messages, parser.ParsedMessage{Ordinal: 2, Role: parser.RoleUser, Content: "Continue implementing this change"}) + a, ar := f.accept(t, "device-a", "recent-a", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, outcome)) + first, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + var pending *string + var outcomeName string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT outcome,signals_pending_since::text FROM sessions WHERE id=$1`, first.SessionID).Scan(&outcomeName, &pending)) + assert.Equal(t, "unknown", outcomeName) + require.NotNil(t, pending) + firstPending := *pending + now = now.Add(time.Minute) + a, ar = f.accept(t, "device-a", "recent-a-again", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, outcome)) + recent, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, first.SessionID, recent.SessionID) + assert.Equal(t, first.CorpusRevision, recent.CorpusRevision, "recent no-op receipt") + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT signals_pending_since::text FROM sessions WHERE id=$1`, first.SessionID).Scan(&pending)) + require.NotNil(t, pending) + assert.Equal(t, firstPending, *pending) + b, br := f.accept(t, "device-b", "equal-recent-b", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, outcome)) + shared, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, shared.State) + var outbox int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_embedding_outbox`).Scan(&outbox)) + now = ended.Add(11 * time.Minute) + a, ar = f.accept(t, "device-a", "settled-a", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, outcome)) + settled, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, shared.SessionID, settled.SessionID) + assert.Equal(t, shared.IdentityRevision, settled.IdentityRevision) + assert.Equal(t, shared.CorpusRevision+1, settled.CorpusRevision) + var score *int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT outcome,signals_pending_since::text,health_score FROM sessions WHERE id=$1`, settled.SessionID).Scan(&outcomeName, &pending, &score)) + assert.Equal(t, "abandoned", outcomeName) + assert.Nil(t, pending) + require.NotNil(t, score) + assert.Less(t, *score, 100) + var afterOutbox int + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT count(*) FROM raw_embedding_outbox`).Scan(&afterOutbox)) + assert.Equal(t, outbox, afterOutbox) + // Remove all source proof, then recreate from the durable shared revision. + empty := outcome + empty.Outcome.Results = nil + a, ar = f.accept(t, "device-a", "gone-a", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, empty)) + b, _ = f.accept(t, "device-b", "gone-b", br.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, b), b, empty)) + a, _ = f.accept(t, "device-a", "return-a", ar.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, a), a, outcome)) + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT outcome,signals_pending_since::text FROM sessions WHERE id=$1`, settled.SessionID).Scan(&outcomeName, &pending)) + assert.Equal(t, "abandoned", outcomeName) + assert.Nil(t, pending) +} diff --git a/internal/postgres/raw_projection_upgrade_pgtest_test.go b/internal/postgres/raw_projection_upgrade_pgtest_test.go new file mode 100644 index 0000000000..19888f03ee --- /dev/null +++ b/internal/postgres/raw_projection_upgrade_pgtest_test.go @@ -0,0 +1,37 @@ +//go:build pgtest + +package postgres + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHostedReprovisionRestoresMigratedColumnsAndIndexes(t *testing.T) { + f := newHostedFixture(t, "tenant-upgrade") + _, err := f.runtime.ExecContext(t.Context(), `INSERT INTO sessions(id,project,machine,agent) VALUES('preserved','project','device','codex')`) + require.NoError(t, err) + _, err = f.admin.ExecContext(t.Context(), `ALTER TABLE sessions DROP COLUMN raw_content_revision; ALTER TABLE raw_ingest_jobs DROP COLUMN projection_selected; ALTER TABLE raw_content_revisions DROP COLUMN recency_state; DROP INDEX idx_tool_result_events_terminal; DROP INDEX raw_session_sources_source; DROP INDEX raw_session_sources_content; DROP INDEX raw_session_sources_physical`) + require.NoError(t, err) + _, constructorErr := NewRawProjectionStore(f.runtime, RawProjectionOptions{Tenant: f.tenant}) + require.Error(t, constructorErr) + assert.Contains(t, constructorErr.Error(), "provisioning") + require.NoError(t, EnsureHostedTenant(t.Context(), f.admin, f.schema, f.tenant)) + var project, revision, tenant string + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT project,raw_content_revision,tenant_id FROM sessions WHERE id='preserved'`).Scan(&project, &revision, &tenant)) + assert.Equal(t, "project", project) + assert.Empty(t, revision) + assert.Equal(t, f.tenant, tenant) + var exists bool + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT to_regclass('idx_tool_result_events_terminal') IS NOT NULL`).Scan(&exists)) + assert.True(t, exists) + for _, index := range []string{"raw_session_sources_source", "raw_session_sources_content", "raw_session_sources_physical"} { + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT to_regclass($1) IS NOT NULL`, index).Scan(&exists)) + assert.True(t, exists, index) + } + _, constructorErr = NewRawProjectionStore(f.runtime, RawProjectionOptions{Tenant: f.tenant}) + require.NoError(t, constructorErr) + require.NoError(t, CheckHostedTenant(t.Context(), f.runtime, f.schema, f.tenant)) +} diff --git a/internal/postgres/raw_projection_worker_pgtest_test.go b/internal/postgres/raw_projection_worker_pgtest_test.go new file mode 100644 index 0000000000..19cb116743 --- /dev/null +++ b/internal/postgres/raw_projection_worker_pgtest_test.go @@ -0,0 +1,93 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +type projectionQueueObserver struct { + rawderive.JobQueue + retries, failures atomic.Int32 +} + +func (q *projectionQueueObserver) RetryRawParseJob(context.Context, rawderive.JobLease, time.Time, string, string) error { + q.retries.Add(1) + return fmt.Errorf("unexpected second retry write") +} +func (q *projectionQueueObserver) FailRawParseJob(context.Context, rawderive.JobLease, string, string) error { + q.failures.Add(1) + return fmt.Errorf("unexpected second failure write") +} + +type projectionPartialParser struct { + body string + root string +} + +func (p *projectionPartialParser) Parse(_ context.Context, m rawsync.CanonicalManifest, tree *rawderive.Materialization) (rawderive.ParsedManifest, error) { + p.root = tree.Root() + body, err := os.ReadFile(filepath.Join(tree.Root(), m.Manifest.Entries[0].Path)) + if err != nil { + return rawderive.ParsedManifest{}, err + } + if string(body) != p.body { + return rawderive.ParsedManifest{}, fmt.Errorf("wrong custody bytes") + } + out := projectionOutcome("proven member") + out.Outcome.ResultSetComplete = false + return out, nil +} +func TestRawProjectionWorkerReportsCommittedPartialOutcomes(t *testing.T) { + for _, maximum := range []int{1, 3} { + t.Run(fmt.Sprint(maximum), func(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "partial-worker", "") + policy := rawderive.RetryPolicy{Base: 2 * time.Second, Maximum: time.Minute, MaxAttempts: maximum} + sink, err := NewRawProjectionStore(f.runtime, RawProjectionOptions{Tenant: f.tenant, RetryPolicy: policy}) + require.NoError(t, err) + _, err = sink.SelectSourceGeneration(t.Context(), m, "parser-1") + require.NoError(t, err) + queue := &projectionQueueObserver{JobQueue: f.jobs} + parser := &projectionPartialParser{body: "synthetic custody partial-worker"} + worker, err := rawderive.NewWorker(rawderive.WorkerConfig{Queue: queue, Manifests: rawderive.ManifestLoader{Store: f.objects, Limits: rawsync.DefaultManifestLimits()}, Materializer: &rawderive.Materializer{Store: f.objects, BaseDir: t.TempDir(), MaxTotalBytes: 4096}, Parser: parser, Projection: sink, Owner: "worker", BatchSize: 1, LeaseDuration: time.Minute, HeartbeatInterval: 10 * time.Second, AttemptTimeout: 30 * time.Second, RetryBase: policy.Base, RetryMax: policy.Maximum, MaxAttempts: policy.MaxAttempts}) + require.NoError(t, err) + result, err := worker.RunBatch(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, result.Claimed) + assert.Zero(t, result.Succeeded) + assert.Zero(t, result.LeaseLost) + assert.Zero(t, queue.retries.Load()) + assert.Zero(t, queue.failures.Load()) + var state string + var delay float64 + require.NoError(t, f.runtime.QueryRowContext(t.Context(), `SELECT state,extract(epoch FROM available_at-clock_timestamp()) FROM raw_ingest_jobs WHERE manifest_id=$1`, m.ManifestID).Scan(&state, &delay)) + if maximum == 1 { + assert.Equal(t, 1, result.Failed) + assert.Zero(t, result.Retried) + assert.Equal(t, "failed", state) + } else { + assert.Equal(t, 1, result.Retried) + assert.Zero(t, result.Failed) + assert.Equal(t, "retrying", state) + assert.Greater(t, delay, 1.0) + assert.LessOrEqual(t, delay, 2.0) + } + r, err := sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, RawIdentityUnique, r.State) + _, err = os.Stat(parser.root) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + } +} diff --git a/internal/postgres/raw_runtime.go b/internal/postgres/raw_runtime.go new file mode 100644 index 0000000000..8de46d989e --- /dev/null +++ b/internal/postgres/raw_runtime.go @@ -0,0 +1,180 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "slices" + "strings" + + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/rawsync" + "go.kenn.io/agentsview/internal/signals" +) + +const pendingRawSignalsSQL = `SELECT id,raw_group_id FROM sessions WHERE tenant_id=$1 AND provenance_kind='raw' AND signals_pending_since IS NOT NULL AND ended_at<=$2 ORDER BY ended_at,id LIMIT $3` + +// SettlePendingSignals examines only a capped indexed set of due, currently +// physical raw revisions. Group locks serialize removal, replacement and +// curation; it never acquires a source-head lock after a group lock. +func (s *RawProjectionStore) SettlePendingSignals(ctx context.Context, limit int) (int, error) { + if limit < 1 || limit > 256 { + return 0, rawsync.ErrInvalid + } + now := s.options.Now() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + rows, err := tx.QueryContext(ctx, pendingRawSignalsSQL, s.options.Tenant, now.Add(-signals.RecencyWindow), limit) + if err != nil { + return 0, err + } + type candidate struct{ id, group string } + var candidates []candidate + var groups []string + for rows.Next() { + var c candidate + if err = rows.Scan(&c.id, &c.group); err != nil { + rows.Close() + return 0, err + } + candidates = append(candidates, c) + groups = append(groups, c.group) + } + err = rows.Err() + rows.Close() + if err != nil { + return 0, err + } + slices.Sort(groups) + groups = slices.Compact(groups) + for _, group := range groups { + var locked string + if err = tx.QueryRowContext(ctx, `SELECT group_id FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, group).Scan(&locked); err != nil { + return 0, err + } + } + changed := 0 + for _, c := range candidates { + var eligible bool + err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sessions s WHERE s.id=$1 AND s.raw_group_id=$2 AND s.provenance_kind='raw' AND s.signals_pending_since IS NOT NULL AND s.ended_at<=$3 AND EXISTS(SELECT 1 FROM raw_session_branches b WHERE b.group_id=$2 AND b.session_id=s.id AND b.active AND NOT COALESCE((SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id=b.branch_id AND c.field='excluded'),(SELECT value='true'::jsonb FROM raw_curation c WHERE c.group_id=b.group_id AND c.branch_id='' AND c.field='excluded'),false)))`, c.id, c.group, now.Add(-signals.RecencyWindow)).Scan(&eligible) + if err != nil { + return 0, err + } + if !eligible { + continue + } + p, err := loadRawPayload(ctx, tx, c.id) + if err != nil { + return 0, err + } + updated, err := publishRawRecency(ctx, tx, c.id, ingest.RefreshSignalRecencyAt(p.Session, p.Messages, p.Signals, now)) + if err != nil { + return 0, err + } + if updated { + changed++ + } + } + if changed > 0 { + if _, err = tx.ExecContext(ctx, `UPDATE raw_corpus_state SET corpus_revision=corpus_revision+1 WHERE singleton=1`); err != nil { + return 0, err + } + } + return changed, tx.Commit() +} + +type RawRolloutBatch struct { + Selected int + Done bool +} + +const rawRolloutHeadsSQL = `SELECT h.device_id,h.provider,h.configured_root_id,h.source_key_sha256,m.manifest_id,m.canonical_json FROM raw_source_heads h JOIN raw_manifests m ON m.tenant_id=h.tenant_id AND m.manifest_id=h.manifest_id WHERE h.tenant_id=$1 AND (h.device_id,h.provider,h.configured_root_id,h.source_key_sha256)>($2,$3,$4,$5) ORDER BY h.device_id,h.provider,h.configured_root_id,h.source_key_sha256 LIMIT $6 FOR UPDATE OF h` + +// ScheduleCurrentHeads is an explicit, resumable version rollout, never an idle +// worker poll. Each call commits at most limit heads and its keyset checkpoint. +// Reusing a run ID requires the same version; equal selections stay completed. +func (s *RawProjectionStore) ScheduleCurrentHeads(ctx context.Context, runID, version string, limit int) (RawRolloutBatch, error) { + if strings.TrimSpace(runID) == "" || len(runID) > 128 || limit < 1 || limit > 256 { + return RawRolloutBatch{}, rawsync.ErrInvalid + } + if err := validateRawIngestProcessingVersion(version); err != nil { + return RawRolloutBatch{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return RawRolloutBatch{}, err + } + defer func() { _ = tx.Rollback() }() + _, err = tx.ExecContext(ctx, `INSERT INTO raw_projection_rollouts(run_id,processing_version) VALUES($1,$2) ON CONFLICT(run_id) DO NOTHING`, runID, version) + if err != nil { + return RawRolloutBatch{}, err + } + var saved, device, provider, root, digest string + var done bool + err = tx.QueryRowContext(ctx, `SELECT processing_version,device_id,provider,configured_root_id,source_key_sha256,complete FROM raw_projection_rollouts WHERE run_id=$1 FOR UPDATE`, runID).Scan(&saved, &device, &provider, &root, &digest, &done) + if err != nil { + return RawRolloutBatch{}, err + } + if saved != version { + return RawRolloutBatch{}, rawsync.ErrConflict + } + if done { + return RawRolloutBatch{Done: true}, nil + } + rows, err := tx.QueryContext(ctx, rawRolloutHeadsSQL, s.options.Tenant, device, provider, root, digest, limit) + if err != nil { + return RawRolloutBatch{}, err + } + type head struct { + device, provider, root, digest, id string + data []byte + } + var heads []head + for rows.Next() { + var h head + if err = rows.Scan(&h.device, &h.provider, &h.root, &h.digest, &h.id, &h.data); err != nil { + rows.Close() + return RawRolloutBatch{}, err + } + heads = append(heads, h) + } + err = rows.Err() + rows.Close() + if err != nil { + return RawRolloutBatch{}, err + } + for _, h := range heads { + m, err := rawsync.ParseCanonicalManifest(rawsync.AuthIdentity{TenantID: s.options.Tenant, DeviceID: h.device}, h.id, h.data, rawsync.DefaultManifestLimits()) + if err != nil { + return RawRolloutBatch{}, err + } + if _, err = selectRawSourceGenerationTx(ctx, tx, m, version); err != nil { + return RawRolloutBatch{}, err + } + device, provider, root, digest = h.device, h.provider, h.root, h.digest + } + done = len(heads) < limit + _, err = tx.ExecContext(ctx, `UPDATE raw_projection_rollouts SET device_id=$2,provider=$3,configured_root_id=$4,source_key_sha256=$5,complete=$6 WHERE run_id=$1`, runID, device, provider, root, digest, done) + if err != nil { + return RawRolloutBatch{}, err + } + return RawRolloutBatch{Selected: len(heads), Done: done}, tx.Commit() +} + +var ErrHostedProjectionOwned = errors.New("pg push cannot mutate a hosted-owned projection") + +// RejectHostedPush guards the database itself, even when pg push used an older +// target configuration that did not declare raw_derivation. +func RejectHostedPush(ctx context.Context, database *sql.DB) error { + var binding sql.NullString + if err := database.QueryRowContext(ctx, `SELECT to_regclass('hosted_tenant_binding')::text`).Scan(&binding); err != nil { + return err + } + if binding.Valid { + return ErrHostedProjectionOwned + } + return nil +} diff --git a/internal/postgres/raw_runtime_indexes_pgtest_test.go b/internal/postgres/raw_runtime_indexes_pgtest_test.go new file mode 100644 index 0000000000..1fabdcb44e --- /dev/null +++ b/internal/postgres/raw_runtime_indexes_pgtest_test.go @@ -0,0 +1,97 @@ +//go:build pgtest + +package postgres + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHostedRuntimeIdleAndMaintenanceQueriesStayBounded(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "current", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("indexed"))) + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-1") + require.NoError(t, err) + small := map[string]float64{} + for _, size := range []int{100, 10000} { + _, err = f.runtime.Exec(`INSERT INTO raw_source_heads(tenant_id,device_id,provider,configured_root_id,source_key,source_key_sha256) SELECT tenant_id,device_id,provider,configured_root_id,'filler-'||n,repeat(md5('filler-'||n),2) FROM raw_manifests CROSS JOIN generate_series(1,$2::int)n WHERE manifest_id=$1 ON CONFLICT DO NOTHING`, m.ManifestID, size) + require.NoError(t, err) + + _, err = f.runtime.Exec(`INSERT INTO raw_manifests(tenant_id,manifest_id,device_id,provider,configured_root_id,source_key,source_key_sha256,capture_id,parent_receipt,receipt,generation,kind,captured_at,canonical_json) + SELECT tenant_id,repeat(md5('filler-'||n),2),device_id,provider,configured_root_id,'filler-'||n,repeat(md5('filler-'||n),2),'capture','',repeat(md5('filler-'||n),2),1,'snapshot',captured_at,canonical_json FROM raw_manifests CROSS JOIN generate_series(1,$2::int)n WHERE manifest_id=$1 ON CONFLICT DO NOTHING`, m.ManifestID, size) + require.NoError(t, err) + _, err = f.runtime.Exec(`INSERT INTO raw_source_heads(tenant_id,device_id,provider,configured_root_id,source_key,source_key_sha256,manifest_id,receipt,generation) + SELECT tenant_id,device_id,provider,configured_root_id,source_key,source_key_sha256,manifest_id,receipt,generation FROM raw_manifests ON CONFLICT(tenant_id,device_id,provider,configured_root_id,source_key_sha256) DO UPDATE SET manifest_id=EXCLUDED.manifest_id,receipt=EXCLUDED.receipt,generation=EXCLUDED.generation; + INSERT INTO raw_ingest_jobs(tenant_id,manifest_id,stage,processing_version,state,projection_generation,available_at) + SELECT tenant_id,manifest_id,'parse','parser-1',CASE WHEN source_key LIKE 'filler-%' THEN 'retrying' ELSE 'complete' END,1,now()+interval '1 day' FROM raw_manifests ON CONFLICT DO NOTHING; + INSERT INTO raw_ingest_jobs(tenant_id,manifest_id,stage,processing_version,state,projection_generation) + SELECT tenant_id,manifest_id,'parse','historical','complete',1 FROM raw_manifests ON CONFLICT DO NOTHING;`) + require.NoError(t, err) + _, err = f.runtime.Exec(`INSERT INTO sessions(id,project,machine,agent,ended_at) SELECT 'settled-'||n,'synthetic','fixture','codex',now()-interval '1 day' FROM generate_series(1,$1::int)n ON CONFLICT DO NOTHING`, size) + require.NoError(t, err) + _, err = f.admin.Exec(`ANALYZE sessions;ANALYZE raw_ingest_jobs;ANALYZE raw_manifests;ANALYZE raw_source_heads;ANALYZE raw_source_projections`) + require.NoError(t, err) + query, args := jobs.rawParseClaimStatement("idle", 1, time.Minute) + for _, tc := range []struct { + name, query string + args []any + }{ + {"claim", query, args}, + {"pending", pendingRawSignalsSQL, []any{f.tenant, time.Now(), 1}}, + {"rollout", rawRolloutHeadsSQL, []any{f.tenant, "", "", "", "", 1}}, + } { + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + tx, err := f.runtime.BeginTx(t.Context(), nil) + require.NoError(t, err) + _, err = tx.Exec(`SET LOCAL plan_cache_mode=` + mode) + require.NoError(t, err) + // A tiny table may reasonably use a sequential scan. Show the indexed + // path there; the large fixture must choose its normal bounded plan. + if size == 100 { + _, err = tx.Exec(`SET LOCAL enable_seqscan=off`) + require.NoError(t, err) + } + var encoded []byte + err = tx.QueryRowContext(t.Context(), `EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) `+tc.query, tc.args...).Scan(&encoded) + require.NoError(t, err) + require.NoError(t, tx.Rollback()) + var plans []struct{ Plan map[string]any } + require.NoError(t, json.Unmarshal(encoded, &plans)) + require.Len(t, plans, 1) + plan := plans[0].Plan + buffers := plan["Shared Hit Blocks"].(float64) + plan["Shared Read Blocks"].(float64) + name := tc.name + "/" + mode + if size == 100 { + small[name] = buffers + } else { + assert.LessOrEqual(t, buffers, small[name]+30, name) + assert.Less(t, buffers, float64(150), name) + } + var visit func(map[string]any) + visit = func(node map[string]any) { + if removed, ok := node["Rows Removed by Filter"].(float64); ok { + assert.LessOrEqual(t, removed, float64(3), name) + } + if children, ok := node["Plans"].([]any); ok { + for _, child := range children { + visit(child.(map[string]any)) + } + } + } + visit(plan) + t.Logf("size=%d %s shared buffers=%.0f", size, name, buffers) + } + } + leases, err := jobs.ClaimRawParseJobs(t.Context(), "idle", 1, time.Minute) + require.NoError(t, err) + assert.Empty(t, leases) + var generation int + require.NoError(t, f.runtime.QueryRow(`SELECT projection_generation FROM raw_source_projections WHERE source_id=$1`, rawSourceID(m)).Scan(&generation)) + assert.Equal(t, 1, generation) + } +} diff --git a/internal/postgres/raw_runtime_pgtest_test.go b/internal/postgres/raw_runtime_pgtest_test.go new file mode 100644 index 0000000000..851d13d954 --- /dev/null +++ b/internal/postgres/raw_runtime_pgtest_test.go @@ -0,0 +1,345 @@ +//go:build pgtest + +package postgres + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawderive" + "go.kenn.io/agentsview/internal/rawsync" +) + +func TestHostedRuntimeAcceptanceSelectsAtomically(t *testing.T) { + f := newProjectionFixture(t) + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-1") + require.NoError(t, err) + f.jobs = jobs + f.custody, err = rawsync.NewService(f.objects, jobs, rawsync.DefaultManifestLimits(), "parser-1") + require.NoError(t, err) + m, receipt := f.accept(t, "device-a", "capture-a", "") + leases, err := jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + assert.EqualValues(t, 1, leases[0].ProjectionGeneration) + require.NoError(t, f.sink.Project(t.Context(), leases[0], m, projectionOutcome("hello"))) + next, _ := f.accept(t, "device-a", "capture-b", receipt.Receipt) + _, err = f.custody.CommitManifest(t.Context(), m.Identity, m.Manifest) + require.NoError(t, err) + leases, err = jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + assert.Equal(t, next.ManifestID, leases[0].ManifestID) + assert.EqualValues(t, 2, leases[0].ProjectionGeneration) +} + +func TestHostedRuntimeClaimsOnlyConfiguredSelection(t *testing.T) { + f := newProjectionFixture(t) + m, _ := f.accept(t, "device-a", "capture-a", "") + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-2") + require.NoError(t, err) + leases, err := jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + assert.Empty(t, leases) + _, err = f.sink.SelectSourceGeneration(t.Context(), m, "parser-1") + require.NoError(t, err) + leases, err = jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + assert.Empty(t, leases) + _, err = f.sink.SelectSourceGeneration(t.Context(), m, "parser-2") + require.NoError(t, err) + leases, err = jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + assert.Equal(t, "parser-2", leases[0].ProcessingVersion) + assert.EqualValues(t, 2, leases[0].ProjectionGeneration) +} + +func TestHostedRuntimeSettlesWithoutUpload(t *testing.T) { + f := newProjectionFixture(t) + ended := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) + now := ended.Add(time.Minute) + f.sink.options.Now = func() time.Time { return now } + out := projectionOutcome("Implement the change") + out.Outcome.Results[0].Result.Session.EndedAt = ended + out.Outcome.Results[0].Result.Messages = append(out.Outcome.Results[0].Result.Messages, parser.ParsedMessage{Ordinal: 2, Role: parser.RoleUser, Content: "Continue implementing this change"}) + m, _ := f.accept(t, "device-a", "recent", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "starred", true)) + before, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + var outbox int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM raw_embedding_outbox`).Scan(&outbox)) + n, err := f.sink.SettlePendingSignals(t.Context(), 1) + require.NoError(t, err) + assert.Zero(t, n) + now = ended.Add(11 * time.Minute) + _, err = f.sink.SelectSourceGeneration(t.Context(), m, "parser-2") + require.NoError(t, err) + n, err = f.sink.SettlePendingSignals(t.Context(), 1) + require.NoError(t, err) + assert.Equal(t, 1, n) + after, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Equal(t, before.SessionID, after.SessionID) + assert.Equal(t, before.IdentityRevision, after.IdentityRevision) + assert.Equal(t, before.CorpusRevision+1, after.CorpusRevision) + var pending *string + var starred bool + var count int + require.NoError(t, f.runtime.QueryRow(`SELECT signals_pending_since,EXISTS(SELECT 1 FROM starred_sessions WHERE session_id=$1) FROM sessions WHERE id=$1`, after.SessionID).Scan(&pending, &starred)) + assert.Nil(t, pending) + assert.True(t, starred) + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM raw_embedding_outbox`).Scan(&count)) + assert.Equal(t, outbox, count) + n, err = f.sink.SettlePendingSignals(t.Context(), 1) + require.NoError(t, err) + assert.Zero(t, n) +} + +func TestHostedRuntimeRolloutResumesAndDoesNotResurrect(t *testing.T) { + f := newProjectionFixture(t) + a, _ := f.accept(t, "device-a", "a", "") + f.accept(t, "device-b", "b", "") + first, err := f.sink.ScheduleCurrentHeads(t.Context(), "rollout", "parser-2", 1) + require.NoError(t, err) + assert.Equal(t, 1, first.Selected) + second, err := f.sink.ScheduleCurrentHeads(t.Context(), "rollout", "parser-2", 1) + require.NoError(t, err) + assert.Equal(t, 1, second.Selected) + end, err := f.sink.ScheduleCurrentHeads(t.Context(), "rollout", "parser-2", 1) + require.NoError(t, err) + assert.True(t, end.Done) + assert.Zero(t, end.Selected) + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-2") + require.NoError(t, err) + leases, err := jobs.ClaimRawParseJobs(t.Context(), "runtime", 1, time.Minute) + require.NoError(t, err) + require.Len(t, leases, 1) + require.Equal(t, a.ManifestID, leases[0].ManifestID) + require.NoError(t, f.sink.Project(t.Context(), leases[0], a, projectionOutcome("hello"))) + _, err = f.sink.ScheduleCurrentHeads(t.Context(), "repeat", "parser-2", 10) + require.NoError(t, err) + var state string + require.NoError(t, f.runtime.QueryRow(`SELECT state FROM raw_ingest_jobs WHERE id=$1`, leases[0].ID).Scan(&state)) + assert.Equal(t, "complete", state) +} + +func TestHostedRuntimeSelectionFailureRollsBackAcceptance(t *testing.T) { + f := newProjectionFixture(t) + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-1") + require.NoError(t, err) + f.custody, err = rawsync.NewService(f.objects, jobs, rawsync.DefaultManifestLimits(), "parser-1") + require.NoError(t, err) + m, r := f.accept(t, "device-a", "first", "") + _, err = f.admin.Exec(`CREATE FUNCTION reject_selection() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic selection dependency unavailable'; END $$; CREATE TRIGGER reject_selection BEFORE INSERT ON raw_projection_generations FOR EACH ROW EXECUTE FUNCTION reject_selection()`) + require.NoError(t, err) + next := m.Manifest + next.CaptureID = "second" + next.ExpectedParentReceipt = r.Receipt + _, err = f.custody.CommitManifest(t.Context(), m.Identity, next) + require.Error(t, err) + for _, table := range []string{"raw_manifests", "raw_ingest_jobs", "raw_projection_generations"} { + var count int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM `+table).Scan(&count)) + assert.Equal(t, 1, count, table) + } + var head string + var generation int + require.NoError(t, f.runtime.QueryRow(`SELECT manifest_id,generation FROM raw_source_heads`).Scan(&head, &generation)) + assert.Equal(t, m.ManifestID, head) + assert.Equal(t, 1, generation) +} + +func TestHostedRuntimeRefusesLegacyReaderAndPush(t *testing.T) { + f := newProjectionFixture(t) + require.ErrorContains(t, RejectHostedPush(t.Context(), f.runtime), "hosted-owned") + _, err := NewStore(f.dsn, f.schema, false) + require.ErrorContains(t, err, "raw_tenant") + require.NoError(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentFull)) + _, err = f.admin.Exec(`REVOKE UPDATE ON raw_content_revisions FROM "` + f.role + `"`) + require.NoError(t, err) + require.ErrorContains(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentFull), "privileges") +} + +func TestHostedRuntimeSettlingRechecksConcurrentVisibility(t *testing.T) { + for _, mode := range []string{"removal", "exclusion", "replacement", "curation"} { + t.Run(mode, func(t *testing.T) { + f := newProjectionFixture(t) + ended := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) + now := ended.Add(time.Minute) + f.sink.options.Now = func() time.Time { return now } + out := projectionOutcome("Implement the change") + out.Outcome.Results[0].Result.Session.EndedAt = ended + out.Outcome.Results[0].Result.Messages = append(out.Outcome.Results[0].Result.Messages, parser.ParsedMessage{Ordinal: 2, Role: parser.RoleUser, Content: "Continue"}) + m, receipt := f.accept(t, "device-a", "initial", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + old, err := f.sink.Resolve(t.Context(), "codex:portable") + require.NoError(t, err) + now = ended.Add(11 * time.Minute) + var action func() error + switch mode { + case "removal", "replacement": + next, _ := f.accept(t, "device-a", "next", receipt.Receipt) + lease := f.lease(t, next) + changed := out + if mode == "removal" { + changed.Outcome.Results = nil + } else { + changed = projectionOutcome("new content") + } + action = func() error { return f.sink.Project(t.Context(), lease, next, changed) } + case "exclusion": + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "trashed", true)) + action = func() error { _, err := f.sink.ExcludeTrashedSession(t.Context(), "codex:portable"); return err } + case "curation": + action = func() error { return f.sink.SetCuration(t.Context(), "codex:portable", "starred", true) } + } + gate, err := f.runtime.BeginTx(t.Context(), nil) + require.NoError(t, err) + defer gate.Rollback() + _, err = gate.Exec(`SELECT group_id FROM raw_session_groups WHERE group_id=$1 FOR UPDATE`, old.GroupID) + require.NoError(t, err) + waitForLocks := func(n int) { + require.Eventually(t, func() bool { + var count int + err := f.admin.QueryRow(`SELECT count(*) FROM pg_stat_activity WHERE usename=$1 AND wait_event_type='Lock'`, f.role).Scan(&count) + return err == nil && count >= n + }, 3*time.Second, 10*time.Millisecond) + } + changed := make(chan error, 1) + go func() { changed <- action() }() + waitForLocks(1) + type result struct { + n int + err error + } + settled := make(chan result, 1) + go func() { n, err := f.sink.SettlePendingSignals(t.Context(), 1); settled <- result{n, err} }() + waitForLocks(2) + require.NoError(t, gate.Commit()) + require.NoError(t, <-changed) + got := <-settled + require.NoError(t, got.err) + if mode == "curation" { + assert.Equal(t, 1, got.n) + stars, err := (&Store{pg: f.runtime}).ListStarredSessionIDs(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{old.SessionID}, stars) + } else { + assert.Zero(t, got.n) + var count int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM sessions WHERE id=$1`, old.SessionID).Scan(&count)) + assert.Zero(t, count) + } + }) + } +} + +func TestHostedRuntimeSubprocessTombstoneRemovesPublicSource(t *testing.T) { + f := newProjectionFixture(t) + jobs, err := NewHostedRawIngestStore(f.runtime, f.tenant, "parser-1") + require.NoError(t, err) + f.jobs = jobs + f.custody, err = rawsync.NewService(f.objects, jobs, rawsync.DefaultManifestLimits(), "parser-1") + require.NoError(t, err) + m, receipt := f.accept(t, "device-a", "initial", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("published"))) + public, err := NewHostedStore(f.dsn, f.schema, f.tenant, false) + require.NoError(t, err) + defer public.Close() + before, err := public.ListSessions(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + require.Len(t, before.Sessions, 1) + tombstone := m.Manifest + tombstone.Kind = rawsync.ManifestTombstone + tombstone.Entries = nil + tombstone.CaptureID = "removed" + tombstone.ExpectedParentReceipt = receipt.Receipt + _, err = f.custody.CommitManifest(t.Context(), m.Identity, tombstone) + require.NoError(t, err) + canonical, err := rawsync.ValidateAndCanonicalize(m.Identity, tombstone, rawsync.DefaultManifestLimits()) + require.NoError(t, err) + isolated, err := rawderive.NewSubprocessParser(time.Second) + require.NoError(t, err) + worker, err := rawderive.NewWorker(rawderive.WorkerConfig{Queue: f.jobs, Manifests: rawderive.ManifestLoader{Store: f.objects, Limits: rawsync.DefaultManifestLimits()}, Materializer: rawderive.Materializer{Store: f.objects, BaseDir: t.TempDir(), MaxTotalBytes: 1}, Parser: isolated, Projection: f.sink, Owner: "tombstone-worker", BatchSize: 1, LeaseDuration: time.Minute, HeartbeatInterval: time.Second, AttemptTimeout: time.Second, RetryBase: time.Second, RetryMax: time.Minute, MaxAttempts: 2}) + require.NoError(t, err) + result, err := worker.RunBatch(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, result.Succeeded) + assert.Zero(t, result.Retried) + var active, proof int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM raw_session_branches WHERE active`).Scan(&active)) + assert.Zero(t, active) + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM session_sources`).Scan(&proof)) + assert.Zero(t, proof) + after, err := public.ListSessions(t.Context(), db.SessionFilter{Limit: 10}) + require.NoError(t, err) + assert.Empty(t, after.Sessions) + gone, err := public.GetSession(t.Context(), "codex:portable") + require.NoError(t, err) + assert.Nil(t, gone) + var state string + require.NoError(t, f.runtime.QueryRow(`SELECT state FROM raw_ingest_jobs WHERE manifest_id=$1`, canonical.ManifestID).Scan(&state)) + assert.Equal(t, "complete", state) +} + +func TestHostedRuntimeAcceptsOnlyRequiredProjectionMutations(t *testing.T) { + f := newProjectionFixture(t) + // These immutable payload/metadata tables require insertion and reads, not + // arbitrary update/delete grants. Child removal follows the sessions FK. + for _, table := range []string{"raw_projection_generations", "raw_source_contributions", "raw_session_public_aliases", "raw_embedding_outbox", "messages", "tool_calls", "tool_result_events", "usage_events", "secret_findings"} { + _, err := f.admin.Exec(`REVOKE UPDATE,DELETE ON "` + table + `" FROM "` + f.role + `"`) + require.NoError(t, err) + } + for _, table := range []string{"tool_calls", "tool_result_events", "usage_events", "pinned_messages"} { + _, err := f.admin.Exec(`REVOKE USAGE ON SEQUENCE "` + table + `_id_seq" FROM "` + f.role + `"; GRANT UPDATE ON SEQUENCE "` + table + `_id_seq" TO "` + f.role + `"`) + require.NoError(t, err) + } + _, err := f.admin.Exec(`REVOKE USAGE ON SEQUENCE secret_findings_id_seq FROM "` + f.role + `"`) + require.NoError(t, err) + require.NoError(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentFull)) + m, receipt := f.accept(t, "device-a", "first", "") + out := projectionOutcome("first") + out.Outcome.Results[0].Result.Messages[1].ToolCalls[0].InputJSON = `{"token":"AKIA7QHWN2DKR4FYPLJM"}` + out.Outcome.Results[0].Result.Messages[1].ToolCalls[0].ResultEvents = []parser.ParsedToolResultEvent{{ToolUseID: "call-1", Source: "tool_result", Status: "completed", Content: "result"}} + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, out)) + for _, table := range []string{"tool_result_events", "usage_events", "secret_findings"} { + var count int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM `+table).Scan(&count)) + assert.Positive(t, count, table) + } + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "starred", true)) + require.NoError(t, f.sink.SetPin(t.Context(), "codex:portable", 0, true, "note")) + n, _ := f.accept(t, "device-a", "next", receipt.Receipt) + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, n), n, projectionOutcome("replacement"))) + require.NoError(t, f.sink.SetCuration(t.Context(), "codex:portable", "trashed", true)) + removed, err := f.sink.ExcludeTrashedSession(t.Context(), "codex:portable") + require.NoError(t, err) + assert.True(t, removed) + var count int + require.NoError(t, f.runtime.QueryRow(`SELECT count(*) FROM sessions`).Scan(&count)) + assert.Zero(t, count) +} + +func TestHostedRuntimeUsagePolicyRequiresVectorRemovalPrivileges(t *testing.T) { + f := newProjectionFixture(t) + _, err := f.admin.Exec(`REVOKE DELETE ON vector_documents FROM "` + f.role + `"`) + require.NoError(t, err) + require.NoError(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentFull)) + require.ErrorContains(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentUsage), "vector_documents") + _, err = f.admin.Exec(`GRANT DELETE ON vector_documents TO "` + f.role + `"`) + require.NoError(t, err) + require.NoError(t, CheckHostedRuntimeWritable(t.Context(), f.runtime, f.schema, config.ArchiveContentUsage)) + f.sink.options.Content.ArchiveContent = config.ArchiveContentUsage + m, _ := f.accept(t, "device-a", "usage", "") + require.NoError(t, f.sink.Project(t.Context(), f.lease(t, m), m, projectionOutcome("discarded transcript"))) + var content string + require.NoError(t, f.runtime.QueryRow(`SELECT content FROM messages ORDER BY ordinal LIMIT 1`).Scan(&content)) + assert.Empty(t, content) +} diff --git a/internal/postgres/raw_runtime_privileges.go b/internal/postgres/raw_runtime_privileges.go new file mode 100644 index 0000000000..45a584c1d4 --- /dev/null +++ b/internal/postgres/raw_runtime_privileges.go @@ -0,0 +1,130 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "go.kenn.io/agentsview/internal/config" +) + +// These are the grants used by Project, SettlePendingSignals, generation +// selection/rollout and supported raw/legacy curation. Cascading session deletes +// do not require DELETE on child payload tables. Row locks on raw_session_groups +// require UPDATE even though its immutable group identity is never rewritten. +var hostedProjectionPrivileges = []struct{ table, privileges string }{ + {"sessions", "SELECT,INSERT,UPDATE,DELETE"}, + {"messages", "SELECT,INSERT"}, {"tool_calls", "SELECT,INSERT"}, + {"tool_result_events", "SELECT,INSERT"}, {"usage_events", "SELECT,INSERT"}, + {"secret_findings", "SELECT,INSERT"}, {"excluded_sessions", "SELECT,INSERT"}, + {"session_aliases", "SELECT"}, {"starred_sessions", "SELECT,INSERT,DELETE"}, + {"pinned_messages", "SELECT,INSERT,UPDATE,DELETE"}, + {"raw_source_projections", "SELECT,INSERT,UPDATE"}, + {"raw_projection_generations", "SELECT,INSERT"}, + {"raw_session_groups", "SELECT,INSERT,UPDATE"}, + {"raw_content_revisions", "SELECT,INSERT,UPDATE"}, + {"raw_session_branches", "SELECT,INSERT,UPDATE"}, + {"session_sources", "SELECT,INSERT,UPDATE,DELETE"}, + {"raw_source_contributions", "SELECT,INSERT"}, + {"raw_session_public_aliases", "SELECT,INSERT"}, + {"raw_curation", "SELECT,INSERT,UPDATE,DELETE"}, + {"raw_pins", "SELECT,INSERT,UPDATE,DELETE"}, + {"raw_corpus_state", "SELECT,INSERT,UPDATE"}, + {"raw_embedding_outbox", "SELECT,INSERT"}, + {"raw_session_links", "SELECT,INSERT,DELETE"}, + {"raw_projection_rollouts", "SELECT,INSERT,UPDATE"}, +} + +// CheckHostedRuntimeWritable is a read-only preflight, never provisioning. +// The configured content policy determines whether the shared usage-only writer +// also needs to remove previously stored vector content. +func CheckHostedRuntimeWritable(ctx context.Context, database *sql.DB, schema string, archive config.ArchiveContent) error { + writable, err := CanWriteRawSyncSchema(ctx, database, schema) + if err != nil { + return err + } + if !writable { + return errors.New("hosted runtime requires writable tenant storage") + } + var readOnly string + if err = database.QueryRowContext(ctx, `SHOW transaction_read_only`).Scan(&readOnly); err != nil { + return err + } + if readOnly != "off" { + return errors.New("hosted runtime requires writable tenant storage") + } + for _, grant := range hostedProjectionPrivileges { + if err = checkHostedTablePrivileges(ctx, database, schema, grant.table, grant.privileges); err != nil { + return err + } + } + // nextval accepts USAGE or UPDATE. GENERATED AS IDENTITY does not require + // sequence privileges; neither does a provisioned sequence-free ID default. + for _, table := range []string{"tool_calls", "tool_result_events", "usage_events", "pinned_messages"} { + var allowed bool + err = database.QueryRowContext(ctx, `SELECT a.attidentity<>'' OR pg_get_serial_sequence(format('%I.%I',$1::text,$2::text),'id') IS NULL OR has_sequence_privilege(pg_get_serial_sequence(format('%I.%I',$1::text,$2::text),'id'),'USAGE,UPDATE') FROM pg_attribute a WHERE a.attrelid=to_regclass(format('%I.%I',$1::text,$2::text)) AND a.attname='id' AND NOT a.attisdropped`, schema, table).Scan(&allowed) + if err != nil { + return err + } + if !allowed { + return fmt.Errorf("hosted runtime sequence privileges for %s are incomplete; owner provisioning required", table) + } + } + if archive.UsageOnly() { + return checkHostedUsageVectorPrivileges(ctx, database, schema) + } + return nil +} + +func checkHostedTablePrivileges(ctx context.Context, database *sql.DB, schema, table, privileges string) error { + var allowed bool + err := database.QueryRowContext(ctx, `SELECT bool_and(has_table_privilege(format('%I.%I',$1::text,$2::text),p)) FROM unnest(string_to_array($3::text,',')) p`, schema, table, privileges).Scan(&allowed) + if err != nil { + return err + } + if !allowed { + return fmt.Errorf("hosted runtime projection privileges for %s are incomplete; owner provisioning required", table) + } + return nil +} + +func checkHostedUsageVectorPrivileges(ctx context.Context, database *sql.DB, schema string) error { + var exists bool + if err := database.QueryRowContext(ctx, `SELECT to_regclass(format('%I.vector_documents',$1::text)) IS NOT NULL`, schema).Scan(&exists); err != nil { + return err + } + if !exists { + return nil + } + for _, grant := range []struct{ table, privileges string }{{"vector_generations", "SELECT"}, {"vector_documents", "SELECT,DELETE"}, {"vector_push_state", "SELECT,DELETE"}} { + if err := checkHostedTablePrivileges(ctx, database, schema, grant.table, grant.privileges); err != nil { + return err + } + } + // Match clearSessionVectorsTx's current-generation table selection exactly. + rows, err := database.QueryContext(ctx, `SELECT 'vector_chunks_g'||id FROM vector_generations WHERE to_regclass(format('%I.%I',$1::text,'vector_chunks_g'||id)) IS NOT NULL`, schema) + if err != nil { + return err + } + var tables []string + for rows.Next() { + var table string + if err = rows.Scan(&table); err != nil { + rows.Close() + return err + } + tables = append(tables, table) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, table := range tables { + if err = checkHostedTablePrivileges(ctx, database, schema, table, "SELECT,DELETE"); err != nil { + return err + } + } + return nil +} diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index 0bd4d3ae76..fa17cbaa36 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -936,443 +936,7 @@ func EnsureSchema( ) // Idempotent column additions for forward compatibility. - alters := []columnMigration{ - { - "model_pricing", "cache_creation_1h_microdollars_per_mtok", - `cache_creation_1h_microdollars_per_mtok BIGINT NOT NULL DEFAULT 0`, - "adding model_pricing.cache_creation_1h_microdollars_per_mtok", - }, - { - "model_pricing_bands", "cache_creation_1h_microdollars_per_mtok", - `cache_creation_1h_microdollars_per_mtok BIGINT NOT NULL DEFAULT 0`, - "adding model_pricing_bands.cache_creation_1h_microdollars_per_mtok", - }, - { - "sessions", "transcript_revision", - `transcript_revision TEXT NOT NULL DEFAULT '0'`, - "adding sessions.transcript_revision", - }, - { - "sessions", "owner_marker", - `owner_marker TEXT NOT NULL DEFAULT ''`, - "adding sessions.owner_marker", - }, - { - "sessions", "deleted_at", - `deleted_at TIMESTAMPTZ`, - "adding sessions.deleted_at", - }, - { - "sessions", "created_at", - `created_at TIMESTAMPTZ`, - "adding sessions.created_at", - }, - { - "sessions", "source_display_name", - `source_display_name TEXT`, - "adding sessions.source_display_name", - }, - { - "sessions", "source_deleted_at", - `source_deleted_at TIMESTAMPTZ`, - "adding sessions.source_deleted_at", - }, - { - "sessions", "deletion_cause", - `deletion_cause TEXT`, - "adding sessions.deletion_cause", - }, - { - "sessions", "parser_parent_session_id", - `parser_parent_session_id TEXT`, - "adding sessions.parser_parent_session_id", - }, - { - "sessions", "total_output_tokens", - `total_output_tokens INT NOT NULL DEFAULT 0`, - "adding sessions.total_output_tokens", - }, - { - "sessions", "peak_context_tokens", - `peak_context_tokens INT NOT NULL DEFAULT 0`, - "adding sessions.peak_context_tokens", - }, - { - "sessions", "has_total_output_tokens", - `has_total_output_tokens BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.has_total_output_tokens", - }, - { - "sessions", "has_peak_context_tokens", - `has_peak_context_tokens BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.has_peak_context_tokens", - }, - { - "messages", "model", - `model TEXT NOT NULL DEFAULT ''`, - "adding messages.model", - }, - { - "messages", "reasoning_effort", - `reasoning_effort TEXT NOT NULL DEFAULT ''`, - "adding messages.reasoning_effort", - }, - { - "messages", "token_usage", - `token_usage TEXT NOT NULL DEFAULT ''`, - "adding messages.token_usage", - }, - { - "messages", "context_tokens", - `context_tokens INT NOT NULL DEFAULT 0`, - "adding messages.context_tokens", - }, - { - "messages", "output_tokens", - `output_tokens INT NOT NULL DEFAULT 0`, - "adding messages.output_tokens", - }, - { - "messages", "provider_id", - `provider_id TEXT NOT NULL DEFAULT ''`, - "adding messages.provider_id", - }, - { - "usage_events", "provider_id", - `provider_id TEXT NOT NULL DEFAULT ''`, - "adding usage_events.provider_id", - }, - { - "messages", "has_context_tokens", - `has_context_tokens BOOLEAN NOT NULL DEFAULT FALSE`, - "adding messages.has_context_tokens", - }, - { - "messages", "has_output_tokens", - `has_output_tokens BOOLEAN NOT NULL DEFAULT FALSE`, - "adding messages.has_output_tokens", - }, - { - "messages", "claude_message_id", - `claude_message_id TEXT NOT NULL DEFAULT ''`, - "adding messages.claude_message_id", - }, - { - "messages", "claude_request_id", - `claude_request_id TEXT NOT NULL DEFAULT ''`, - "adding messages.claude_request_id", - }, - { - "tool_calls", "call_index", - `call_index INT NOT NULL DEFAULT 0`, - "adding tool_calls.call_index", - }, - { - "tool_calls", "file_path", - `file_path TEXT`, - "adding tool_calls.file_path", - }, - { - "sessions", "prompt_evidence_discarded", - `prompt_evidence_discarded BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.prompt_evidence_discarded", - }, - { - "sessions", "is_automated", - `is_automated BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.is_automated", - }, - { - "sessions", "tool_failure_signal_count", - `tool_failure_signal_count INT NOT NULL DEFAULT 0`, - "adding sessions.tool_failure_signal_count", - }, - { - "sessions", "tool_retry_count", - `tool_retry_count INT NOT NULL DEFAULT 0`, - "adding sessions.tool_retry_count", - }, - { - "sessions", "edit_churn_count", - `edit_churn_count INT NOT NULL DEFAULT 0`, - "adding sessions.edit_churn_count", - }, - { - "sessions", "consecutive_failure_max", - `consecutive_failure_max INT NOT NULL DEFAULT 0`, - "adding sessions.consecutive_failure_max", - }, - { - "sessions", "outcome", - `outcome TEXT NOT NULL DEFAULT 'unknown'`, - "adding sessions.outcome", - }, - { - "sessions", "outcome_confidence", - `outcome_confidence TEXT NOT NULL DEFAULT 'low'`, - "adding sessions.outcome_confidence", - }, - { - "sessions", "ended_with_role", - `ended_with_role TEXT NOT NULL DEFAULT ''`, - "adding sessions.ended_with_role", - }, - { - "sessions", "final_failure_streak", - `final_failure_streak INT NOT NULL DEFAULT 0`, - "adding sessions.final_failure_streak", - }, - { - "sessions", "signals_pending_since", - `signals_pending_since TEXT`, - "adding sessions.signals_pending_since", - }, - { - "sessions", "compaction_count", - `compaction_count INT NOT NULL DEFAULT 0`, - "adding sessions.compaction_count", - }, - { - "sessions", "mid_task_compaction_count", - `mid_task_compaction_count INT NOT NULL DEFAULT 0`, - "adding sessions.mid_task_compaction_count", - }, - { - "sessions", "context_pressure_max", - `context_pressure_max DOUBLE PRECISION`, - "adding sessions.context_pressure_max", - }, - { - "sessions", "health_score", - `health_score INT`, - "adding sessions.health_score", - }, - { - "sessions", "health_grade", - `health_grade TEXT`, - "adding sessions.health_grade", - }, - { - "sessions", "has_tool_calls", - `has_tool_calls BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.has_tool_calls", - }, - { - "sessions", "has_context_data", - `has_context_data BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.has_context_data", - }, - { - "sessions", "quality_signal_version", - `quality_signal_version INT NOT NULL DEFAULT 0`, - "adding sessions.quality_signal_version", - }, - { - "sessions", "short_prompt_count", - `short_prompt_count INT NOT NULL DEFAULT 0`, - "adding sessions.short_prompt_count", - }, - { - "sessions", "unstructured_start", - `unstructured_start BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.unstructured_start", - }, - { - "sessions", "missing_success_criteria_count", - `missing_success_criteria_count INT NOT NULL DEFAULT 0`, - "adding sessions.missing_success_criteria_count", - }, - { - "sessions", "missing_verification_count", - `missing_verification_count INT NOT NULL DEFAULT 0`, - "adding sessions.missing_verification_count", - }, - { - "sessions", "duplicate_prompt_count", - `duplicate_prompt_count INT NOT NULL DEFAULT 0`, - "adding sessions.duplicate_prompt_count", - }, - { - "sessions", "no_code_context_count", - `no_code_context_count INT NOT NULL DEFAULT 0`, - "adding sessions.no_code_context_count", - }, - { - "sessions", "runaway_tool_loop_count", - `runaway_tool_loop_count INT NOT NULL DEFAULT 0`, - "adding sessions.runaway_tool_loop_count", - }, - { - "sessions", "data_version", - `data_version INT NOT NULL DEFAULT 0`, - "adding sessions.data_version", - }, - { - "sessions", "cwd", - `cwd TEXT NOT NULL DEFAULT ''`, - "adding sessions.cwd", - }, - { - "sessions", "git_branch", - `git_branch TEXT NOT NULL DEFAULT ''`, - "adding sessions.git_branch", - }, - { - "sessions", "source_session_id", - `source_session_id TEXT NOT NULL DEFAULT ''`, - "adding sessions.source_session_id", - }, - { - "sessions", "source_version", - `source_version TEXT NOT NULL DEFAULT ''`, - "adding sessions.source_version", - }, - { - "sessions", "transcript_fidelity", - `transcript_fidelity TEXT NOT NULL DEFAULT ''`, - "adding sessions.transcript_fidelity", - }, - { - "sessions", "parser_malformed_lines", - `parser_malformed_lines INT NOT NULL DEFAULT 0`, - "adding sessions.parser_malformed_lines", - }, - { - "sessions", "is_truncated", - `is_truncated BOOLEAN NOT NULL DEFAULT FALSE`, - "adding sessions.is_truncated", - }, - { - "messages", "source_type", - `source_type TEXT NOT NULL DEFAULT ''`, - "adding messages.source_type", - }, - { - "messages", "source_subtype", - `source_subtype TEXT NOT NULL DEFAULT ''`, - "adding messages.source_subtype", - }, - { - "messages", "prompt_source", - `prompt_source TEXT NOT NULL DEFAULT ''`, - "adding messages.prompt_source", - }, - { - "messages", "source_uuid", - `source_uuid TEXT NOT NULL DEFAULT ''`, - "adding messages.source_uuid", - }, - { - "messages", "source_parent_uuid", - `source_parent_uuid TEXT NOT NULL DEFAULT ''`, - "adding messages.source_parent_uuid", - }, - { - "messages", "is_sidechain", - `is_sidechain BOOLEAN NOT NULL DEFAULT FALSE`, - "adding messages.is_sidechain", - }, - { - "messages", "is_compact_boundary", - `is_compact_boundary BOOLEAN NOT NULL DEFAULT FALSE`, - "adding messages.is_compact_boundary", - }, - { - "messages", "thinking_text", - `thinking_text TEXT NOT NULL DEFAULT ''`, - "adding messages.thinking_text", - }, - { - "sessions", "termination_status", - `termination_status TEXT`, - "adding sessions.termination_status", - }, - { - "sessions", "secret_leak_count", - `secret_leak_count INTEGER NOT NULL DEFAULT 0`, - "adding sessions.secret_leak_count", - }, - { - "sessions", "secrets_rules_version", - `secrets_rules_version TEXT NOT NULL DEFAULT ''`, - "adding sessions.secrets_rules_version", - }, - { - "sessions", "session_name", - `session_name TEXT`, - "adding sessions.session_name", - }, - { - "sessions", "agent_label", - `agent_label TEXT NOT NULL DEFAULT ''`, - "adding sessions.agent_label", - }, - { - "sessions", "entrypoint", - `entrypoint TEXT NOT NULL DEFAULT ''`, - "adding sessions.entrypoint", - }, - { - "sessions", "session_kind", - `session_kind TEXT NOT NULL DEFAULT ''`, - "adding sessions.session_kind", - }, - { - "source_project_identity_observations", "source_archive_id", - `source_archive_id TEXT NOT NULL DEFAULT ''`, - "adding source_project_identity_observations.source_archive_id", - }, - { - "source_project_identity_observations", "source_archive_salt", - `source_archive_salt TEXT NOT NULL DEFAULT ''`, - "adding source_project_identity_observations.source_archive_salt", - }, - { - "source_project_identity_observations", "repository_path", - `repository_path TEXT NOT NULL DEFAULT ''`, - "adding source_project_identity_observations.repository_path", - }, - { - "source_project_identity_observations", "worktree_relationship", - `worktree_relationship TEXT NOT NULL DEFAULT 'unknown'`, - "adding source_project_identity_observations.worktree_relationship", - }, - { - "source_project_identity_observations", "checkout_state", - `checkout_state TEXT NOT NULL DEFAULT 'unknown'`, - "adding source_project_identity_observations.checkout_state", - }, - { - "source_project_identity_observations", "git_branch", - `git_branch TEXT NOT NULL DEFAULT ''`, - "adding source_project_identity_observations.git_branch", - }, - { - "source_project_identity_observations", "remote_resolution", - `remote_resolution TEXT NOT NULL DEFAULT 'unknown'`, - "adding source_project_identity_observations.remote_resolution", - }, - { - "source_project_identity_observations", "remote_candidate_count", - `remote_candidate_count INT NOT NULL DEFAULT 0`, - "adding source_project_identity_observations.remote_candidate_count", - }, - { - "sessions", "source_archive_id", - `source_archive_id TEXT NOT NULL DEFAULT ''`, - "adding sessions.source_archive_id", - }, - { - "sessions", "source_database_generation", - `source_database_generation TEXT NOT NULL DEFAULT ''`, - "adding sessions.source_database_generation", - }, - { - "sessions", "file_path", - `file_path TEXT`, - "adding sessions.file_path", - }, - } + alters := schemaColumnMigrations() step = time.Now() existingColumns, err := loadExistingColumns( ctx, db, alters, @@ -1533,8 +1097,9 @@ func EnsureSchema( // createPartialIndexesPG creates partial indexes on the PG schema. // Idempotent via IF NOT EXISTS. -func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { +func createPartialIndexesPG(ctx context.Context, db schemaExecer) error { indexes := []string{ + `CREATE INDEX IF NOT EXISTS idx_sessions_raw_group ON sessions(raw_group_id) WHERE raw_group_id <> ''`, // Match SQLite's bounded Activity terminal-event lookup, including // completions that outlive a session's ended_at metadata. `CREATE INDEX IF NOT EXISTS idx_tool_result_events_terminal @@ -1959,6 +1524,10 @@ func batchUpdateAutomatedPG( return nil } +type schemaExecer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + type columnQueryer interface { QueryContext(context.Context, string, ...any) (*sql.Rows, error) } @@ -2029,7 +1598,7 @@ func loadExistingColumns( } func ensureColumns( - ctx context.Context, db *sql.DB, + ctx context.Context, db schemaExecer, existing map[string]map[string]bool, migrations []columnMigration, ) ([]string, error) { diff --git a/internal/postgres/schema_columns.go b/internal/postgres/schema_columns.go new file mode 100644 index 0000000000..fd3f03f100 --- /dev/null +++ b/internal/postgres/schema_columns.go @@ -0,0 +1,445 @@ +package postgres + +// schemaColumnMigrations is shared by legacy setup and atomic hosted provisioning. +func schemaColumnMigrations() []columnMigration { + return []columnMigration{ + {"sessions", "provenance_kind", `provenance_kind TEXT NOT NULL DEFAULT 'legacy'`, "adding sessions.provenance_kind"}, + {"sessions", "raw_group_id", `raw_group_id TEXT NOT NULL DEFAULT ''`, "adding sessions.raw_group_id"}, + {"sessions", "raw_content_revision", `raw_content_revision TEXT NOT NULL DEFAULT ''`, "adding sessions.raw_content_revision"}, + { + "model_pricing", "cache_creation_1h_microdollars_per_mtok", + `cache_creation_1h_microdollars_per_mtok BIGINT NOT NULL DEFAULT 0`, + "adding model_pricing.cache_creation_1h_microdollars_per_mtok", + }, + { + "model_pricing_bands", "cache_creation_1h_microdollars_per_mtok", + `cache_creation_1h_microdollars_per_mtok BIGINT NOT NULL DEFAULT 0`, + "adding model_pricing_bands.cache_creation_1h_microdollars_per_mtok", + }, + { + "sessions", "transcript_revision", + `transcript_revision TEXT NOT NULL DEFAULT '0'`, + "adding sessions.transcript_revision", + }, + { + "sessions", "owner_marker", + `owner_marker TEXT NOT NULL DEFAULT ''`, + "adding sessions.owner_marker", + }, + { + "sessions", "deleted_at", + `deleted_at TIMESTAMPTZ`, + "adding sessions.deleted_at", + }, + { + "sessions", "created_at", + `created_at TIMESTAMPTZ`, + "adding sessions.created_at", + }, + { + "sessions", "source_display_name", + `source_display_name TEXT`, + "adding sessions.source_display_name", + }, + { + "sessions", "source_deleted_at", + `source_deleted_at TIMESTAMPTZ`, + "adding sessions.source_deleted_at", + }, + { + "sessions", "deletion_cause", + `deletion_cause TEXT`, + "adding sessions.deletion_cause", + }, + { + "sessions", "parser_parent_session_id", + `parser_parent_session_id TEXT`, + "adding sessions.parser_parent_session_id", + }, + { + "sessions", "total_output_tokens", + `total_output_tokens INT NOT NULL DEFAULT 0`, + "adding sessions.total_output_tokens", + }, + { + "sessions", "peak_context_tokens", + `peak_context_tokens INT NOT NULL DEFAULT 0`, + "adding sessions.peak_context_tokens", + }, + { + "sessions", "has_total_output_tokens", + `has_total_output_tokens BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.has_total_output_tokens", + }, + { + "sessions", "has_peak_context_tokens", + `has_peak_context_tokens BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.has_peak_context_tokens", + }, + { + "messages", "model", + `model TEXT NOT NULL DEFAULT ''`, + "adding messages.model", + }, + { + "messages", "reasoning_effort", + `reasoning_effort TEXT NOT NULL DEFAULT ''`, + "adding messages.reasoning_effort", + }, + { + "messages", "token_usage", + `token_usage TEXT NOT NULL DEFAULT ''`, + "adding messages.token_usage", + }, + { + "messages", "context_tokens", + `context_tokens INT NOT NULL DEFAULT 0`, + "adding messages.context_tokens", + }, + { + "messages", "output_tokens", + `output_tokens INT NOT NULL DEFAULT 0`, + "adding messages.output_tokens", + }, + { + "messages", "provider_id", + `provider_id TEXT NOT NULL DEFAULT ''`, + "adding messages.provider_id", + }, + { + "usage_events", "provider_id", + `provider_id TEXT NOT NULL DEFAULT ''`, + "adding usage_events.provider_id", + }, + { + "messages", "has_context_tokens", + `has_context_tokens BOOLEAN NOT NULL DEFAULT FALSE`, + "adding messages.has_context_tokens", + }, + { + "messages", "has_output_tokens", + `has_output_tokens BOOLEAN NOT NULL DEFAULT FALSE`, + "adding messages.has_output_tokens", + }, + { + "messages", "claude_message_id", + `claude_message_id TEXT NOT NULL DEFAULT ''`, + "adding messages.claude_message_id", + }, + { + "messages", "claude_request_id", + `claude_request_id TEXT NOT NULL DEFAULT ''`, + "adding messages.claude_request_id", + }, + { + "tool_calls", "call_index", + `call_index INT NOT NULL DEFAULT 0`, + "adding tool_calls.call_index", + }, + { + "tool_calls", "file_path", + `file_path TEXT`, + "adding tool_calls.file_path", + }, + { + "sessions", "prompt_evidence_discarded", + `prompt_evidence_discarded BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.prompt_evidence_discarded", + }, + { + "sessions", "is_automated", + `is_automated BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.is_automated", + }, + { + "sessions", "tool_failure_signal_count", + `tool_failure_signal_count INT NOT NULL DEFAULT 0`, + "adding sessions.tool_failure_signal_count", + }, + { + "sessions", "tool_retry_count", + `tool_retry_count INT NOT NULL DEFAULT 0`, + "adding sessions.tool_retry_count", + }, + { + "sessions", "edit_churn_count", + `edit_churn_count INT NOT NULL DEFAULT 0`, + "adding sessions.edit_churn_count", + }, + { + "sessions", "consecutive_failure_max", + `consecutive_failure_max INT NOT NULL DEFAULT 0`, + "adding sessions.consecutive_failure_max", + }, + { + "sessions", "outcome", + `outcome TEXT NOT NULL DEFAULT 'unknown'`, + "adding sessions.outcome", + }, + { + "sessions", "outcome_confidence", + `outcome_confidence TEXT NOT NULL DEFAULT 'low'`, + "adding sessions.outcome_confidence", + }, + { + "sessions", "ended_with_role", + `ended_with_role TEXT NOT NULL DEFAULT ''`, + "adding sessions.ended_with_role", + }, + { + "sessions", "final_failure_streak", + `final_failure_streak INT NOT NULL DEFAULT 0`, + "adding sessions.final_failure_streak", + }, + { + "sessions", "signals_pending_since", + `signals_pending_since TEXT`, + "adding sessions.signals_pending_since", + }, + { + "sessions", "compaction_count", + `compaction_count INT NOT NULL DEFAULT 0`, + "adding sessions.compaction_count", + }, + { + "sessions", "mid_task_compaction_count", + `mid_task_compaction_count INT NOT NULL DEFAULT 0`, + "adding sessions.mid_task_compaction_count", + }, + { + "sessions", "context_pressure_max", + `context_pressure_max DOUBLE PRECISION`, + "adding sessions.context_pressure_max", + }, + { + "sessions", "health_score", + `health_score INT`, + "adding sessions.health_score", + }, + { + "sessions", "health_grade", + `health_grade TEXT`, + "adding sessions.health_grade", + }, + { + "sessions", "has_tool_calls", + `has_tool_calls BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.has_tool_calls", + }, + { + "sessions", "has_context_data", + `has_context_data BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.has_context_data", + }, + { + "sessions", "quality_signal_version", + `quality_signal_version INT NOT NULL DEFAULT 0`, + "adding sessions.quality_signal_version", + }, + { + "sessions", "short_prompt_count", + `short_prompt_count INT NOT NULL DEFAULT 0`, + "adding sessions.short_prompt_count", + }, + { + "sessions", "unstructured_start", + `unstructured_start BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.unstructured_start", + }, + { + "sessions", "missing_success_criteria_count", + `missing_success_criteria_count INT NOT NULL DEFAULT 0`, + "adding sessions.missing_success_criteria_count", + }, + { + "sessions", "missing_verification_count", + `missing_verification_count INT NOT NULL DEFAULT 0`, + "adding sessions.missing_verification_count", + }, + { + "sessions", "duplicate_prompt_count", + `duplicate_prompt_count INT NOT NULL DEFAULT 0`, + "adding sessions.duplicate_prompt_count", + }, + { + "sessions", "no_code_context_count", + `no_code_context_count INT NOT NULL DEFAULT 0`, + "adding sessions.no_code_context_count", + }, + { + "sessions", "runaway_tool_loop_count", + `runaway_tool_loop_count INT NOT NULL DEFAULT 0`, + "adding sessions.runaway_tool_loop_count", + }, + { + "sessions", "data_version", + `data_version INT NOT NULL DEFAULT 0`, + "adding sessions.data_version", + }, + { + "sessions", "cwd", + `cwd TEXT NOT NULL DEFAULT ''`, + "adding sessions.cwd", + }, + { + "sessions", "git_branch", + `git_branch TEXT NOT NULL DEFAULT ''`, + "adding sessions.git_branch", + }, + { + "sessions", "source_session_id", + `source_session_id TEXT NOT NULL DEFAULT ''`, + "adding sessions.source_session_id", + }, + { + "sessions", "source_version", + `source_version TEXT NOT NULL DEFAULT ''`, + "adding sessions.source_version", + }, + { + "sessions", "transcript_fidelity", + `transcript_fidelity TEXT NOT NULL DEFAULT ''`, + "adding sessions.transcript_fidelity", + }, + { + "sessions", "parser_malformed_lines", + `parser_malformed_lines INT NOT NULL DEFAULT 0`, + "adding sessions.parser_malformed_lines", + }, + { + "sessions", "is_truncated", + `is_truncated BOOLEAN NOT NULL DEFAULT FALSE`, + "adding sessions.is_truncated", + }, + { + "messages", "source_type", + `source_type TEXT NOT NULL DEFAULT ''`, + "adding messages.source_type", + }, + { + "messages", "source_subtype", + `source_subtype TEXT NOT NULL DEFAULT ''`, + "adding messages.source_subtype", + }, + { + "messages", "prompt_source", + `prompt_source TEXT NOT NULL DEFAULT ''`, + "adding messages.prompt_source", + }, + { + "messages", "source_uuid", + `source_uuid TEXT NOT NULL DEFAULT ''`, + "adding messages.source_uuid", + }, + { + "messages", "source_parent_uuid", + `source_parent_uuid TEXT NOT NULL DEFAULT ''`, + "adding messages.source_parent_uuid", + }, + { + "messages", "is_sidechain", + `is_sidechain BOOLEAN NOT NULL DEFAULT FALSE`, + "adding messages.is_sidechain", + }, + { + "messages", "is_compact_boundary", + `is_compact_boundary BOOLEAN NOT NULL DEFAULT FALSE`, + "adding messages.is_compact_boundary", + }, + { + "messages", "thinking_text", + `thinking_text TEXT NOT NULL DEFAULT ''`, + "adding messages.thinking_text", + }, + { + "sessions", "termination_status", + `termination_status TEXT`, + "adding sessions.termination_status", + }, + { + "sessions", "secret_leak_count", + `secret_leak_count INTEGER NOT NULL DEFAULT 0`, + "adding sessions.secret_leak_count", + }, + { + "sessions", "secrets_rules_version", + `secrets_rules_version TEXT NOT NULL DEFAULT ''`, + "adding sessions.secrets_rules_version", + }, + { + "sessions", "session_name", + `session_name TEXT`, + "adding sessions.session_name", + }, + { + "sessions", "agent_label", + `agent_label TEXT NOT NULL DEFAULT ''`, + "adding sessions.agent_label", + }, + { + "sessions", "entrypoint", + `entrypoint TEXT NOT NULL DEFAULT ''`, + "adding sessions.entrypoint", + }, + { + "sessions", "session_kind", + `session_kind TEXT NOT NULL DEFAULT ''`, + "adding sessions.session_kind", + }, + { + "source_project_identity_observations", "source_archive_id", + `source_archive_id TEXT NOT NULL DEFAULT ''`, + "adding source_project_identity_observations.source_archive_id", + }, + { + "source_project_identity_observations", "source_archive_salt", + `source_archive_salt TEXT NOT NULL DEFAULT ''`, + "adding source_project_identity_observations.source_archive_salt", + }, + { + "source_project_identity_observations", "repository_path", + `repository_path TEXT NOT NULL DEFAULT ''`, + "adding source_project_identity_observations.repository_path", + }, + { + "source_project_identity_observations", "worktree_relationship", + `worktree_relationship TEXT NOT NULL DEFAULT 'unknown'`, + "adding source_project_identity_observations.worktree_relationship", + }, + { + "source_project_identity_observations", "checkout_state", + `checkout_state TEXT NOT NULL DEFAULT 'unknown'`, + "adding source_project_identity_observations.checkout_state", + }, + { + "source_project_identity_observations", "git_branch", + `git_branch TEXT NOT NULL DEFAULT ''`, + "adding source_project_identity_observations.git_branch", + }, + { + "source_project_identity_observations", "remote_resolution", + `remote_resolution TEXT NOT NULL DEFAULT 'unknown'`, + "adding source_project_identity_observations.remote_resolution", + }, + { + "source_project_identity_observations", "remote_candidate_count", + `remote_candidate_count INT NOT NULL DEFAULT 0`, + "adding source_project_identity_observations.remote_candidate_count", + }, + { + "sessions", "source_archive_id", + `source_archive_id TEXT NOT NULL DEFAULT ''`, + "adding sessions.source_archive_id", + }, + { + "sessions", "source_database_generation", + `source_database_generation TEXT NOT NULL DEFAULT ''`, + "adding sessions.source_database_generation", + }, + { + "sessions", "file_path", + `file_path TEXT`, + "adding sessions.file_path", + }, + } +} diff --git a/internal/postgres/schema_test.go b/internal/postgres/schema_test.go index bfacb53786..82a0f3e9ad 100644 --- a/internal/postgres/schema_test.go +++ b/internal/postgres/schema_test.go @@ -1176,6 +1176,7 @@ func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { // source_project_identity_observations: repository/worktree/checkout/remote // context). Per-table batching means one ALTER each. tool_calls // lists all its migration columns (call_index, file_path) as present, so - // it contributes no ALTER. + // it contributes no ALTER. Already-present raw job columns are probed + // without issuing a redundant ALTER. assert.Equal(t, 4, state.alterTableExecCount(), "ALTER TABLE execs") } diff --git a/internal/postgres/session_timing.go b/internal/postgres/session_timing.go index 1cee2fcff3..7001258d4b 100644 --- a/internal/postgres/session_timing.go +++ b/internal/postgres/session_timing.go @@ -100,6 +100,9 @@ func (s *Store) queryTurnRows( func (s *Store) queryCallRows( ctx context.Context, sessionID string, ) ([]db.CallRow, error) { + return s.queryCallRowsWithJoin(ctx, sessionID, `LEFT JOIN sessions s_sub ON s_sub.id = tc.subagent_session_id`) +} +func (s *Store) queryCallRowsWithJoin(ctx context.Context, sessionID, subagentJoin string) ([]db.CallRow, error) { now := time.Now().UTC() rows, err := s.pg.QueryContext(ctx, ` SELECT @@ -143,8 +146,7 @@ func (s *Store) queryCallRows( ELSE NULL END AS subagent_duration_ms FROM tool_calls tc - LEFT JOIN sessions s_sub - ON s_sub.id = tc.subagent_session_id + `+subagentJoin+` WHERE tc.session_id = $2 ORDER BY tc.message_ordinal, tc.id `, now, sessionID) diff --git a/internal/postgres/sessions.go b/internal/postgres/sessions.go index e4a32d39e7..99ac483f71 100644 --- a/internal/postgres/sessions.go +++ b/internal/postgres/sessions.go @@ -19,9 +19,10 @@ import ( // Store wraps a PostgreSQL connection for read-only session // queries. type Store struct { - pg *sql.DB - cursorMu sync.RWMutex - cursorSecret []byte + hostedRelations bool + pg *sql.DB + cursorMu sync.RWMutex + cursorSecret []byte insightCapabilityMu sync.RWMutex insightGenerationAvailable bool @@ -441,7 +442,7 @@ func (s *Store) ListSessions( f.Limit = db.DefaultSessionLimit } - where, args := buildPGSessionFilter(f) + where, args := db.BuildSessionFilterSQL(f, s.sessionDialect()) dialect := db.PostgresQueryDialect() rs := db.ResolveSort(f) @@ -537,7 +538,7 @@ func (s *Store) GetSidebarSessionIndex( rootFilter.IncludeChildren = false rootWhere, rootArgs := buildPGSessionBaseFilter(rootFilter) canonicalRootWhere := db.BuildCanonicalRootWhere( - db.PostgresQueryDialect(), "sessions", f.IncludeOrphans, + s.sessionDialect(), "sessions", f.IncludeOrphans, ) var total int countQuery := "SELECT COUNT(*) FROM sessions WHERE " + @@ -549,7 +550,7 @@ func (s *Store) GetSidebarSessionIndex( fmt.Errorf("counting sidebar roots: %w", err) } - where, args := buildPGSessionFilter(f) + where, args := db.BuildSessionFilterSQL(f, s.sessionDialect()) query := ` SELECT id, @@ -608,7 +609,7 @@ func (s *Store) getSidebarSessionIndexPage( rootFilter.Cursor = "" rootFilter.Starred = false rootWhere, rootArgs := buildPGSessionBaseFilter(rootFilter) - canonicalRootWhere := db.BuildCanonicalRootWhere(db.PostgresQueryDialect(), "sessions", f.IncludeOrphans) + canonicalRootWhere := db.BuildCanonicalRootWhere(s.sessionDialect(), "sessions", f.IncludeOrphans) childAutomationPred := pgAutomatedScopePredicate( normalizePGAutomatedScope(f.AutomatedScope, f.ExcludeAutomated), "s.is_automated", @@ -642,7 +643,7 @@ func (s *Store) getSidebarSessionIndexPage( UNION SELECT t.root_id, s.id FROM sessions s - JOIN tree t ON s.parent_session_id = t.id + JOIN tree t ON ` + s.sessionDialect().ParentRelation("s", "t") + ` WHERE s.message_count > 0 AND s.deleted_at IS NULL ` + childAutomationWhere + ` @@ -692,7 +693,7 @@ func (s *Store) getSidebarSessionIndexPage( UNION SELECT t.root_id, s.id FROM sessions s - JOIN tree t ON s.parent_session_id = t.id + JOIN tree t ON ` + s.sessionDialect().ParentRelation("s", "t") + ` WHERE s.message_count > 0 AND s.deleted_at IS NULL ` + childAutomationWhere + ` @@ -781,7 +782,7 @@ func (s *Store) getSidebarSessionIndexPage( UNION SELECT s.id, t.ord FROM sessions s - JOIN tree t ON s.parent_session_id = t.id + JOIN tree t ON ` + s.sessionDialect().ParentRelation("s", "t") + ` WHERE s.message_count > 0 AND s.deleted_at IS NULL ` + childAutomationWhere + ` diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 0378fd9b91..3ece6dcd3a 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -3,6 +3,7 @@ package postgres import ( "context" "database/sql" + "errors" "fmt" "io" "strings" @@ -26,9 +27,31 @@ func NewStore( if err != nil { return nil, err } + if err = RejectHostedPush(context.Background(), pg); err != nil { + pg.Close() + if errors.Is(err, ErrHostedProjectionOwned) { + return nil, fmt.Errorf("hosted schema requires raw_tenant and the hosted read adapter") + } + return nil, err + } return &Store{pg: pg}, nil } +// NewHostedStore opens the read adapter with the same fixed tenant boundary as +// custody. It fails before serving queries if the runtime role or schema is unsafe. +func NewHostedStore(pgURL, schema, tenant string, allowInsecure bool) (*HostedStore, error) { + pg, err := OpenHosted(pgURL, schema, tenant, allowInsecure) + if err != nil { + return nil, err + } + h, err := newHostedAdapter(pg, tenant) + if err != nil { + pg.Close() + return nil, err + } + return h, nil +} + // DB returns the underlying *sql.DB for operations that need // direct access (e.g. schema compatibility checks). func (s *Store) DB() *sql.DB { return s.pg } @@ -551,6 +574,10 @@ func (s *Store) ListTrashedSessions( // EmptyTrash permanently deletes every trashed session. func (s *Store) EmptyTrash() (int, error) { + return s.emptyTrash(false) +} + +func (s *Store) emptyTrash(legacyOnly bool) (int, error) { ctx := context.Background() tx, err := s.pg.BeginTx(ctx, nil) if err != nil { @@ -558,8 +585,12 @@ func (s *Store) EmptyTrash() (int, error) { } defer func() { _ = tx.Rollback() }() + where := "s.deleted_at IS NOT NULL" + if legacyOnly { + where += " AND s.provenance_kind='legacy'" + } sessionIDs, excludedIDs, err := readPGTrashedSessionExclusions( - ctx, tx, "s.deleted_at IS NOT NULL", + ctx, tx, where, ) if err != nil { return 0, mapPGWriteError("locking trashed sessions", err) @@ -579,7 +610,12 @@ func (s *Store) EmptyTrash() (int, error) { if err != nil { return 0, mapPGWriteError("emptying trash", err) } - if err := deletePGExcludedSessionRows(ctx, tx, excludedIDs); err != nil { + if legacyOnly { + err = deleteLegacyExcludedSessionRows(ctx, tx, excludedIDs) + } else { + err = deletePGExcludedSessionRows(ctx, tx, excludedIDs) + } + if err != nil { return 0, mapPGWriteError("purging excluded trashed session aliases", err) } if err := tx.Commit(); err != nil { diff --git a/internal/postgres/sync.go b/internal/postgres/sync.go index 031682dc18..cc28423675 100644 --- a/internal/postgres/sync.go +++ b/internal/postgres/sync.go @@ -268,6 +268,10 @@ func New( if err != nil { return nil, err } + if err = RejectHostedPush(context.Background(), pg); err != nil { + pg.Close() + return nil, err + } targetFingerprint, err := pgTargetFingerprint(pgURL, schema) if err != nil { pg.Close() diff --git a/internal/rawderive/job.go b/internal/rawderive/job.go index 0abc01257f..4dada80b78 100644 --- a/internal/rawderive/job.go +++ b/internal/rawderive/job.go @@ -30,11 +30,12 @@ func ValidLeaseOwner(owner string) bool { // JobLease is the fenced ownership token for one parse attempt. type JobLease struct { - ID int64 - Identity rawsync.AuthIdentity - ManifestID string - ProcessingVersion string - Attempt int - Owner string - ExpiresAt time.Time + ID int64 + Identity rawsync.AuthIdentity + ManifestID string + ProcessingVersion string + ProjectionGeneration int64 + Attempt int + Owner string + ExpiresAt time.Time } diff --git a/internal/rawderive/retry.go b/internal/rawderive/retry.go new file mode 100644 index 0000000000..0791c2b3df --- /dev/null +++ b/internal/rawderive/retry.go @@ -0,0 +1,53 @@ +package rawderive + +import ( + "fmt" + "time" + + "go.kenn.io/agentsview/internal/rawsync" +) + +// RetryPolicy is shared by ordinary worker failures and transaction-owned +// partial projection outcomes. Runtime must supply the same policy to both. +type RetryPolicy struct { + Base, Maximum time.Duration + MaxAttempts int +} +type RetryDecision struct { + Failed bool + Delay time.Duration +} + +func (p RetryPolicy) Validate() error { + if p.Base <= 0 || p.Maximum < p.Base { + return fmt.Errorf("%w: raw parse worker retry timing is invalid", rawsync.ErrInvalid) + } + if p.MaxAttempts <= 0 { + return fmt.Errorf("%w: raw parse worker max attempts must be positive", rawsync.ErrInvalid) + } + return nil +} +func (p RetryPolicy) Decide(attempt int) RetryDecision { + if attempt >= p.MaxAttempts { + return RetryDecision{Failed: true} + } + return RetryDecision{Delay: retryDelay(attempt, p.Base, p.Maximum)} +} + +// CommittedProjectionOutcome reports an incomplete result whose proven content +// and retry/terminal job state have already committed atomically. The worker +// must not perform ordinary failure bookkeeping or let a later heartbeat error +// override this durable outcome. +type CommittedProjectionOutcome uint8 + +const ( + ProjectionRetrying CommittedProjectionOutcome = iota + 1 + ProjectionFailed +) + +func (o CommittedProjectionOutcome) Error() string { + if o == ProjectionRetrying { + return "raw projection committed; retry scheduled" + } + return "raw projection committed; attempt limit reached" +} diff --git a/internal/rawderive/subprocess.go b/internal/rawderive/subprocess.go new file mode 100644 index 0000000000..f0a498cb49 --- /dev/null +++ b/internal/rawderive/subprocess.go @@ -0,0 +1,340 @@ +package rawderive + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "sync" + "time" + + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawsync" +) + +const ParserChildFlag = "--internal-raw-parser" +const parserOutputLimit = 32 << 20 +const parserErrorLimit = 64 << 10 +const parserProbeFile = "parser-probe" +const parserProbeContent = "hosted parser source probe" + +var ErrSandboxUnavailable = errors.New("hosted parser isolation unavailable; Linux user, mount, network namespaces and seccomp TSYNC are required") +var errParserProtocol = errors.New("parser protocol invalid or output limit exceeded") +var errParserFailed = errors.New("isolated parser failed") + +// SubprocessParser never invokes a provider in the hosting process. Limits are +// fixed hard ceilings; WallTimeout is validated before activation. +type SubprocessParser struct{ WallTimeout time.Duration } + +func NewSubprocessParser(wall time.Duration) (*SubprocessParser, error) { + if wall <= 0 || wall > 5*time.Minute { + return nil, rawsync.ErrInvalid + } + return &SubprocessParser{WallTimeout: wall}, nil +} + +type parserRequest struct { + Identity rawsync.AuthIdentity + ManifestID string + CanonicalJSON []byte + Probe bool +} +type wireSourceError struct { + SourceKey, DisplayPath, SessionID, Code string + Retryable bool +} +type wireMessageState struct { + PresenceKnown bool + TokenUsageNil bool +} +type wireOutcome struct { + Outcome parser.ParseOutcome + Tombstone bool + Errors []wireSourceError + Digests [][]byte + SessionPresence []bool + MessagePresence [][]wireMessageState +} + +func encodeParserOutcome(p ParsedManifest) ([]byte, error) { + w := wireOutcome{Outcome: p.Outcome, Tombstone: p.Tombstone} + w.Outcome.SourceErrors = nil + for _, e := range p.Outcome.SourceErrors { + w.Errors = append(w.Errors, wireSourceError{e.SourceKey, e.DisplayPath, e.SessionID, jobErrorCode(e.Err), e.Retryable}) + } + for _, r := range p.Outcome.Results { + w.SessionPresence = append(w.SessionPresence, r.Result.Session.AggregateTokenPresenceKnown()) + messagePresence := make([]wireMessageState, len(r.Result.Messages)) + for i, m := range r.Result.Messages { + messagePresence[i] = wireMessageState{PresenceKnown: m.TokenPresenceKnown(), TokenUsageNil: m.TokenUsage == nil} + } + w.MessagePresence = append(w.MessagePresence, messagePresence) + for _, m := range r.Result.Messages { + for _, c := range m.ToolCalls { + for _, e := range c.ResultEvents { + w.Digests = append(w.Digests, e.RawContentDigest) + } + } + } + } + return json.Marshal(w) +} +func decodeParserOutcome(data []byte) (ParsedManifest, error) { + var w wireOutcome + if len(data) > parserOutputLimit { + return ParsedManifest{}, errParserProtocol + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&w); err != nil { + return ParsedManifest{}, errParserProtocol + } + var extra any + if dec.Decode(&extra) != io.EOF || len(w.Outcome.SourceErrors) != 0 { + return ParsedManifest{}, errParserProtocol + } + for _, e := range w.Errors { + var err error + switch e.Code { + case "": + case "canceled": + err = context.Canceled + case "deadline_exceeded": + err = context.DeadlineExceeded + case "invalid": + err = rawsync.ErrInvalid + case "internal", "object_not_found", "missing_object", "conflict", "lease_lost": + err = errParserFailed + default: + return ParsedManifest{}, errParserProtocol + } + w.Outcome.SourceErrors = append(w.Outcome.SourceErrors, parser.SourceError{SourceKey: e.SourceKey, DisplayPath: e.DisplayPath, SessionID: e.SessionID, Retryable: e.Retryable, Err: err}) + } + if len(w.SessionPresence) != len(w.Outcome.Results) || len(w.MessagePresence) != len(w.Outcome.Results) { + return ParsedManifest{}, errParserProtocol + } + n := 0 + for i := range w.Outcome.Results { + result := &w.Outcome.Results[i].Result + if len(w.MessagePresence[i]) != len(result.Messages) { + return ParsedManifest{}, errParserProtocol + } + result.Session.RestoreAggregateTokenPresenceKnown(w.SessionPresence[i]) + for j := range result.Messages { + result.Messages[j].RestoreTokenPresenceKnown(w.MessagePresence[i][j].PresenceKnown) + if w.MessagePresence[i][j].TokenUsageNil { + result.Messages[j].TokenUsage = nil + } + } + for j := range w.Outcome.Results[i].Result.Messages { + for k := range w.Outcome.Results[i].Result.Messages[j].ToolCalls { + for l := range w.Outcome.Results[i].Result.Messages[j].ToolCalls[k].ResultEvents { + if n >= len(w.Digests) { + return ParsedManifest{}, errParserProtocol + } + w.Outcome.Results[i].Result.Messages[j].ToolCalls[k].ResultEvents[l].RawContentDigest = w.Digests[n] + n++ + } + } + } + } + if n != len(w.Digests) { + return ParsedManifest{}, errParserProtocol + } + return ParsedManifest{Outcome: w.Outcome, Tombstone: w.Tombstone}, nil +} + +func (p *SubprocessParser) Preflight(ctx context.Context) error { + source, err := os.MkdirTemp("", "raw-probe-") + if err != nil { + return ErrSandboxUnavailable + } + defer os.RemoveAll(source) + if os.WriteFile(filepath.Join(source, parserProbeFile), []byte(parserProbeContent), 0400) != nil { + return ErrSandboxUnavailable + } + _, err = p.execute(ctx, source, parserRequest{Probe: true}) + if err != nil { + return ErrSandboxUnavailable + } + return nil +} +func (p *SubprocessParser) Parse(ctx context.Context, m rawsync.CanonicalManifest, tree *Materialization) (ParsedManifest, error) { + if err := rawsync.ValidateCanonicalManifest(m); err != nil { + return ParsedManifest{}, err + } + if m.Manifest.Kind == rawsync.ManifestTombstone { + return ParsedManifest{Tombstone: true}, nil + } + if tree == nil || tree.Root() == "" { + return ParsedManifest{}, rawsync.ErrInvalid + } + data, err := p.execute(ctx, tree.Root(), parserRequest{Identity: m.Identity, ManifestID: m.ManifestID, CanonicalJSON: m.CanonicalJSON}) + if err != nil { + return ParsedManifest{}, err + } + return decodeParserOutcome(data) +} +func (p *SubprocessParser) execute(parent context.Context, source string, req parserRequest) ([]byte, error) { + input, err := json.Marshal(req) + if err != nil || len(input) > 2*rawsync.DefaultManifestLimits().MaxCanonicalBytes+4096 { + return nil, errParserProtocol + } + jail, err := os.MkdirTemp("", "raw-jail-") + if err != nil { + return nil, ErrSandboxUnavailable + } + defer os.RemoveAll(jail) + if os.Mkdir(filepath.Join(jail, "source"), 0700) != nil { + return nil, ErrSandboxUnavailable + } + exe, err := os.Executable() + if err != nil { + return nil, ErrSandboxUnavailable + } + ctx, cancel := context.WithTimeout(parent, p.WallTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, exe, ParserChildFlag, source, jail) + cmd.Env = []string{"GOMAXPROCS=2", "GOMEMLIMIT=384MiB", "TZ=UTC", "LANG=C"} + if err = configureParserNamespace(cmd); err != nil { + return nil, err + } + return runParserProcess(ctx, cancel, cmd, input, parserOutputLimit, parserErrorLimit) +} + +type parserCapture struct { + data bytes.Buffer + limit int + cancel context.CancelFunc + ready chan struct{} + once sync.Once + overflow bool +} + +func (w *parserCapture) Write(p []byte) (int, error) { + if len(p) > w.limit-w.data.Len() { + w.overflow = true + w.cancel() + return 0, errParserProtocol + } + w.data.Write(p) + if w.ready != nil && w.data.Len() >= 6 { + w.once.Do(func() { close(w.ready) }) + } + return len(p), nil +} + +// runParserProcess retains Wait on every started process, including output +// overflow, protocol failure, cancellation and a provider ignoring context. +func runParserProcess(ctx context.Context, cancel context.CancelFunc, cmd *exec.Cmd, input []byte, outLimit, errLimit int) ([]byte, error) { + ready := make(chan struct{}) + out := &parserCapture{limit: outLimit, cancel: cancel, ready: ready} + diagnostic := &parserCapture{limit: errLimit, cancel: cancel} + cmd.Stdout = out + cmd.Stderr = diagnostic + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errParserFailed + } + if err = cmd.Start(); err != nil { + stdin.Close() + return nil, ErrSandboxUnavailable + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + var waitErr error + select { + case <-ready: + // The bounded writer is still active. Validate its readiness prefix + // after Wait; never read the growing buffer concurrently. + _, err = stdin.Write(input) + stdin.Close() + if err != nil { + cancel() + } + waitErr = <-done + case waitErr = <-done: + stdin.Close() + case <-ctx.Done(): + cancel() + stdin.Close() + waitErr = <-done + } + if out.overflow || diagnostic.overflow { + return nil, errParserProtocol + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if waitErr != nil { + return nil, errParserFailed + } + if !bytes.HasPrefix(out.data.Bytes(), []byte("READY\n")) { + return nil, errParserProtocol + } + return bytes.Clone(out.data.Bytes()[6:]), nil +} + +// RunParserChild must be dispatched before ordinary CLI/config/database setup. +// Only a successful bootstrap may read the untrusted bounded request. +func RunParserChild(args []string) (bool, int) { + if len(args) == 0 || args[0] != ParserChildFlag { + return false, 0 + } + if len(args) != 3 || !parserFDBootstrapCompleted() || isolateParser(args[1], args[2]) != nil { + return true, 78 + } + if _, err := os.Stdout.WriteString("READY\n"); err != nil { + return true, 79 + } + input, err := io.ReadAll(io.LimitReader(os.Stdin, int64(2*rawsync.DefaultManifestLimits().MaxCanonicalBytes+4097))) + if err != nil || len(input) > 2*rawsync.DefaultManifestLimits().MaxCanonicalBytes+4096 { + return true, 79 + } + var req parserRequest + if json.Unmarshal(input, &req) != nil { + return true, 79 + } + if req.Probe { + if verifyParserProbe("/source") != nil { + return true, 78 + } + return true, 0 + } + m, err := rawsync.ParseCanonicalManifest(req.Identity, req.ManifestID, req.CanonicalJSON, rawsync.DefaultManifestLimits()) + if err != nil { + return true, 79 + } + tree := &Materialization{root: "/source", entries: map[string]string{}} + for _, e := range m.Manifest.Entries { + tree.entries[e.Path] = filepath.Join("/source", filepath.FromSlash(e.Path)) + } + provider, err := NewProviderParser(parser.ProviderFactories(), "hosted") + if err != nil { + return true, 79 + } + result, err := provider.Parse(context.Background(), m, tree) + if err != nil { + return true, 79 + } + data, err := encodeParserOutcome(result) + if err != nil || len(data) > parserOutputLimit { + return true, 79 + } + if _, err = os.Stdout.Write(data); err != nil { + return true, 79 + } + return true, 0 +} + +func verifyParserProbe(root string) error { + data, err := os.ReadFile(filepath.Join(root, parserProbeFile)) + if err != nil || string(data) != parserProbeContent { + return ErrSandboxUnavailable + } + return nil +} diff --git a/internal/rawderive/subprocess_bootstrap_linux.go b/internal/rawderive/subprocess_bootstrap_linux.go new file mode 100644 index 0000000000..ccb9700df2 --- /dev/null +++ b/internal/rawderive/subprocess_bootstrap_linux.go @@ -0,0 +1,29 @@ +//go:build linux && cgo && (amd64 || arm64) + +package rawderive + +/* +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +// Run before Go initializes its own poll descriptors. Plain exec preserves +// deliberately non-CLOEXEC descriptors; closing later would also close Go's. +static int raw_parser_fds_closed = 0; +static int raw_parser_fd_bootstrap_completed(void) { return raw_parser_fds_closed; } + +__attribute__((constructor)) static void raw_parser_close_inherited_fds(void) { + const char *mode = getenv("AGENTSVIEW_RAW_PARSER_CHILD"); + if (mode == NULL || strcmp(mode, "1") != 0) return; + if (syscall(SYS_close_range, 3U, UINT_MAX, 0U) != 0) _exit(78); + raw_parser_fds_closed = 1; +} +*/ +import "C" + +const parserFDBootstrapAvailable = true + +func parserFDBootstrapCompleted() bool { return C.raw_parser_fd_bootstrap_completed() == 1 } diff --git a/internal/rawderive/subprocess_bootstrap_nocgo_linux.go b/internal/rawderive/subprocess_bootstrap_nocgo_linux.go new file mode 100644 index 0000000000..6ef15a20e6 --- /dev/null +++ b/internal/rawderive/subprocess_bootstrap_nocgo_linux.go @@ -0,0 +1,7 @@ +//go:build linux && !cgo && (amd64 || arm64) + +package rawderive + +const parserFDBootstrapAvailable = false + +func parserFDBootstrapCompleted() bool { return false } diff --git a/internal/rawderive/subprocess_linux.go b/internal/rawderive/subprocess_linux.go new file mode 100644 index 0000000000..6fb33c3dff --- /dev/null +++ b/internal/rawderive/subprocess_linux.go @@ -0,0 +1,122 @@ +//go:build linux && (amd64 || arm64) + +package rawderive + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +func configureParserNamespace(cmd *exec.Cmd) error { + if !parserFDBootstrapAvailable { + return ErrSandboxUnavailable + } + cmd.Env = append(cmd.Env, "AGENTSVIEW_RAW_PARSER_CHILD=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: unix.CLONE_NEWUSER | unix.CLONE_NEWNS | unix.CLONE_NEWNET, + UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Geteuid(), Size: 1}}, + GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getegid(), Size: 1}}, + GidMappingsEnableSetgroups: false, Setpgid: true, Pdeathsig: syscall.SIGKILL, + } + return nil +} +func isolateParser(source, jail string) error { + if !filepath.IsAbs(source) || !filepath.IsAbs(jail) || source == jail { + return ErrSandboxUnavailable + } + if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { + return err + } + // Bind the jail first: covering it later hides its source submount. + if err := unix.Mount(jail, jail, "", unix.MS_BIND, ""); err != nil { + return err + } + target := filepath.Join(jail, "source") + if err := unix.Mount(source, target, "", unix.MS_BIND, ""); err != nil { + return err + } + flags := uintptr(unix.MS_BIND | unix.MS_REMOUNT | unix.MS_RDONLY | unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC) + if err := unix.Mount("", target, "", flags, ""); err != nil { + return err + } + if err := unix.Mount("", jail, "", flags, ""); err != nil { + return err + } + if err := unix.Chroot(jail); err != nil { + return err + } + if err := unix.Chdir("/"); err != nil { + return err + } + for resource, limit := range map[int]uint64{unix.RLIMIT_AS: 2 << 30, unix.RLIMIT_DATA: 512 << 20, unix.RLIMIT_CPU: 30, unix.RLIMIT_CORE: 0, unix.RLIMIT_NOFILE: 64, unix.RLIMIT_FSIZE: 0} { + if err := unix.Setrlimit(resource, &unix.Rlimit{Cur: limit, Max: limit}); err != nil { + return err + } + } + return installParserSeccomp() +} +func installParserSeccomp() error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + // seccomp_data: nr@0, arch@4, args@16. x32 is rejected even on amd64. + f := []unix.SockFilter{ + {Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 4}, + {Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: parserAuditArch, Jt: 1}, + {Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_KILL_PROCESS}, + {Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 0}, + {Code: unix.BPF_JMP | unix.BPF_JSET | unix.BPF_K, K: 0x40000000, Jf: 1}, + {Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_KILL_PROCESS}, + } + ret := func(nr uint32, action uint32) { + f = append(f, unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: nr, Jf: 1}, unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: action}) + } + ret(unix.SYS_CLONE3, unix.SECCOMP_RET_ERRNO|uint32(unix.ENOSYS)) + // clone is permitted solely for runtime threads, with no namespace or + // process creation flags. Reject high bits as well as unfamiliar low bits. + cloneFlags := uint32(unix.CLONE_VM | unix.CLONE_FS | unix.CLONE_FILES | unix.CLONE_SIGHAND | unix.CLONE_THREAD | unix.CLONE_SYSVSEM | unix.CLONE_SETTLS | unix.CLONE_PARENT_SETTID | unix.CLONE_CHILD_CLEARTID | unix.CLONE_CHILD_SETTID | unix.CLONE_DETACHED) + f = append(f, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: unix.SYS_CLONE, Jf: 9}, + unix.SockFilter{Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 20}, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: 0, Jf: 5}, + unix.SockFilter{Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 16}, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JSET | unix.BPF_K, K: ^cloneFlags, Jt: 3}, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JSET | unix.BPF_K, K: unix.CLONE_THREAD, Jf: 2}, + unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ALLOW}, + unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ERRNO | uint32(unix.EPERM)}, + unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ERRNO | uint32(unix.EPERM)}, + unix.SockFilter{Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 0}, + ) + for _, nr := range []uint32{unix.SYS_KILL, unix.SYS_TGKILL} { + f = append(f, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: nr, Jf: 4}, + unix.SockFilter{Code: unix.BPF_LD | unix.BPF_W | unix.BPF_ABS, K: 16}, + unix.SockFilter{Code: unix.BPF_JMP | unix.BPF_JEQ | unix.BPF_K, K: uint32(os.Getpid()), Jf: 1}, + unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ALLOW}, + unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ERRNO | uint32(unix.EPERM)}, + ) + } + allowed := []uint32{unix.SYS_READ, unix.SYS_WRITE, unix.SYS_READV, unix.SYS_WRITEV, unix.SYS_PREAD64, unix.SYS_CLOSE, unix.SYS_LSEEK, unix.SYS_FSTAT, unix.SYS_NEWFSTATAT, unix.SYS_STATX, unix.SYS_GETDENTS64, unix.SYS_READLINKAT, unix.SYS_OPENAT, unix.SYS_FACCESSAT, unix.SYS_FACCESSAT2, unix.SYS_FCNTL, unix.SYS_FLOCK, unix.SYS_MMAP, unix.SYS_MPROTECT, unix.SYS_MUNMAP, unix.SYS_MADVISE, unix.SYS_MREMAP, unix.SYS_BRK, unix.SYS_FUTEX, unix.SYS_SCHED_YIELD, unix.SYS_SCHED_GETAFFINITY, unix.SYS_CLOCK_GETTIME, unix.SYS_CLOCK_NANOSLEEP, unix.SYS_NANOSLEEP, unix.SYS_GETTIMEOFDAY, unix.SYS_RT_SIGACTION, unix.SYS_RT_SIGPROCMASK, unix.SYS_RT_SIGRETURN, unix.SYS_SIGALTSTACK, unix.SYS_RT_SIGSUSPEND, unix.SYS_RESTART_SYSCALL, unix.SYS_GETPID, unix.SYS_GETPPID, unix.SYS_GETTID, unix.SYS_GETUID, unix.SYS_GETEUID, unix.SYS_GETGID, unix.SYS_GETEGID, unix.SYS_GETRANDOM, unix.SYS_EPOLL_CREATE1, unix.SYS_EPOLL_CTL, unix.SYS_EPOLL_PWAIT, unix.SYS_EVENTFD2, unix.SYS_PIPE2, unix.SYS_DUP, unix.SYS_DUP3, unix.SYS_SET_ROBUST_LIST, unix.SYS_RSEQ, unix.SYS_SET_TID_ADDRESS, unix.SYS_GETCWD, unix.SYS_UNAME, unix.SYS_EXIT, unix.SYS_EXIT_GROUP} + for _, nr := range append(allowed, parserArchSyscalls...) { + ret(nr, unix.SECCOMP_RET_ALLOW) + } + f = append(f, unix.SockFilter{Code: unix.BPF_RET | unix.BPF_K, K: unix.SECCOMP_RET_ERRNO | uint32(unix.EPERM)}) + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + program := unix.SockFprog{Len: uint16(len(f)), Filter: &f[0]} + result, _, errno := unix.Syscall(unix.SYS_SECCOMP, unix.SECCOMP_SET_MODE_FILTER, unix.SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(&program))) + runtime.KeepAlive(f) + if errno != 0 { + return errno + } + if result != 0 { + return ErrSandboxUnavailable + } + return nil +} diff --git a/internal/rawderive/subprocess_linux_amd64.go b/internal/rawderive/subprocess_linux_amd64.go new file mode 100644 index 0000000000..d8bfa78d0e --- /dev/null +++ b/internal/rawderive/subprocess_linux_amd64.go @@ -0,0 +1,7 @@ +package rawderive + +import "golang.org/x/sys/unix" + +const parserAuditArch = unix.AUDIT_ARCH_X86_64 + +var parserArchSyscalls = []uint32{unix.SYS_OPEN, unix.SYS_STAT, unix.SYS_LSTAT, unix.SYS_READLINK, unix.SYS_ACCESS, unix.SYS_EPOLL_WAIT, unix.SYS_POLL, unix.SYS_SELECT, unix.SYS_ARCH_PRCTL, unix.SYS_DUP2} diff --git a/internal/rawderive/subprocess_linux_arm64.go b/internal/rawderive/subprocess_linux_arm64.go new file mode 100644 index 0000000000..cb1b0df7a0 --- /dev/null +++ b/internal/rawderive/subprocess_linux_arm64.go @@ -0,0 +1,7 @@ +package rawderive + +import "golang.org/x/sys/unix" + +const parserAuditArch = unix.AUDIT_ARCH_AARCH64 + +var parserArchSyscalls = []uint32{} diff --git a/internal/rawderive/subprocess_linux_test.go b/internal/rawderive/subprocess_linux_test.go new file mode 100644 index 0000000000..9d7f24165c --- /dev/null +++ b/internal/rawderive/subprocess_linux_test.go @@ -0,0 +1,331 @@ +//go:build linux && (amd64 || arm64) + +package rawderive + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawsync" + "golang.org/x/sys/unix" +) + +func TestMain(m *testing.M) { + if handled, code := RunParserChild(os.Args[1:]); handled { + os.Exit(code) + } + if len(os.Args) > 1 && os.Args[1] == "--sandbox-fd-test" { + if os.Getenv("AGENTSVIEW_RAW_PARSER_CHILD") == "1" && !parserFDBootstrapCompleted() { + os.Exit(81) + } + fd, _ := strconv.Atoi(os.Args[2]) + data := make([]byte, 64) + n, err := unix.Read(fd, data) + if err == nil && string(data[:n]) == "outside descriptor sentinel" { + os.Exit(80) + } + if len(os.Args) > 3 { + sock, _ := strconv.Atoi(os.Args[3]) + if n, e := unix.Write(sock, []byte("outside socket sentinel")); e == nil && n > 0 { + os.Exit(80) + } + } + os.Stdout.WriteString("closed") + os.Exit(0) + } + if len(os.Args) > 1 && os.Args[1] == "--sandbox-kernel-test" { + os.Exit(sandboxKernelHelper(os.Args[2:])) + } + os.Exit(m.Run()) +} +func sandboxKernelHelper(args []string) int { + mode, source, jail, outside := args[0], args[1], args[2], args[3] + // Keep several existing threads live across filter installation. + gate := make(chan struct{}) + results := make(chan bool, 4) + ready := make(chan struct{}, 4) + for range 4 { + go func() { + runtime.LockOSThread() + ready <- struct{}{} + <-gate + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, 0) + if fd >= 0 { + unix.Close(fd) + } + results <- err == unix.EPERM + }() + } + for range 4 { + <-ready + } + var isolationErr error + if mode == "seccomp" { + isolationErr = installParserSeccomp() + } else { + isolationErr = isolateParser(source, jail) + } + if isolationErr != nil { + return 78 + } + os.Stdout.WriteString("READY\n") + switch mode { + case "cpu": + //nolint:staticcheck // SA5002: deliberate CPU burn verifies the child's hard CPU limit. + for { + } + case "memory": + var held [][]byte + for { + b := make([]byte, 8<<20) + for i := range b { + b[i] = 1 + } + held = append(held, b) + runtime.KeepAlive(held) + } + case "denials", "seccomp": + checks := map[string]bool{} + if mode == "denials" { + body, err := os.ReadFile("/source/input") + checks["source_read"] = err == nil && string(body) == "fixture" + _, err = os.ReadFile(outside) + checks["outside_read"] = err != nil + _ = os.Chmod("/source/input", 0600) + checks["source_write"] = os.WriteFile("/source/input", []byte("bad"), 0600) != nil + checks["jail_write"] = os.WriteFile("/escape", []byte("bad"), 0600) != nil + _, err = unix.Mmap(-1, 0, 3<<30, unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON) + checks["address_limit"] = err == unix.ENOMEM + _, err = unix.Mmap(-1, 0, 600<<20, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_PRIVATE|unix.MAP_ANON) + checks["data_limit"] = err == unix.ENOMEM + } + + for _, domain := range []int{unix.AF_INET, unix.AF_INET6, unix.AF_UNIX} { + fd, e := unix.Socket(domain, unix.SOCK_DGRAM, 0) + if fd >= 0 { + unix.Close(fd) + } + checks["socket_"+string(rune('a'+domain))] = e == unix.EPERM + } + close(gate) + for range 4 { + checks["threads"] = <-results + if !checks["threads"] { + return 80 + } + } + fresh := make(chan bool) + go func() { + runtime.LockOSThread() + fd, e := unix.Socket(unix.AF_UNIX, unix.SOCK_STREAM, 0) + if fd >= 0 { + unix.Close(fd) + } + fresh <- e == unix.EPERM + }() + checks["new_thread"] = <-fresh + checks["exec"] = unix.Exec("/source/input", []string{"input"}, nil) == unix.EPERM + pid, _, errno := unix.RawSyscall(unix.SYS_CLONE, uintptr(unix.SIGCHLD), 0, 0) + if errno == 0 && pid == 0 { + os.Exit(90) + } + checks["fork"] = errno == unix.EPERM + checks["unshare"] = unix.Unshare(unix.CLONE_NEWNET) == unix.EPERM + checks["mount"] = unix.Mount("", "/", "", unix.MS_REMOUNT, "") == unix.EPERM + checks["raise_limit"] = unix.Setrlimit(unix.RLIMIT_AS, &unix.Rlimit{Cur: 3 << 30, Max: 3 << 30}) == unix.EPERM + checks["environment"] = os.Getenv("RAW_SANDBOX_SECRET") == "" + checks["foreign_signal"] = unix.Kill(1, 0) == unix.EPERM + json.NewEncoder(os.Stdout).Encode(checks) + return 0 + } + return 80 +} +func TestSandboxKernelControls(t *testing.T) { + p, err := NewSubprocessParser(5 * time.Second) + require.NoError(t, err) + if err = p.Preflight(t.Context()); err != nil { + require.ErrorIs(t, err, ErrSandboxUnavailable) + if os.Getenv("RAW_SANDBOX_REQUIRED") == "1" { + t.Fatal("mandatory kernel isolation unavailable") + } + t.Skip("kernel isolation unavailable; positive controls require Linux namespace support") + } + t.Setenv("RAW_SANDBOX_SECRET", "must-not-inherit") + for _, mode := range []string{"denials", "memory", "cpu"} { + t.Run(mode, func(t *testing.T) { + source := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(source, "input"), []byte("fixture"), 0400)) + jail := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(jail, "source"), 0700)) + outside := filepath.Join(t.TempDir(), "sentinel") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0400)) + ctx, cancel := context.WithTimeout(t.Context(), 40*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "--sandbox-kernel-test", mode, source, jail, outside) + cmd.Env = []string{"GOMAXPROCS=2", "GOMEMLIMIT=384MiB"} + require.NoError(t, configureParserNamespace(cmd)) + start := time.Now() + out, err := runParserProcess(ctx, cancel, cmd, nil, parserOutputLimit, parserErrorLimit) + require.NotNil(t, cmd.ProcessState) + if mode == "denials" { + require.NoError(t, err) + var checks map[string]bool + require.NoError(t, json.Unmarshal(out, &checks)) + require.GreaterOrEqual(t, len(checks), 14) + for key, value := range checks { + assert.True(t, value, key) + } + } else { + require.Error(t, err) + assert.NoError(t, ctx.Err()) + assert.Less(t, time.Since(start), 38*time.Second) + if mode == "cpu" { + status, ok := cmd.ProcessState.Sys().(syscall.WaitStatus) + require.True(t, ok) + assert.Equal(t, syscall.SIGKILL, status.Signal()) + assert.Greater(t, time.Since(start), 20*time.Second) + } else { + assert.Equal(t, 2, cmd.ProcessState.ExitCode(), "Go hard allocation failure") + } + } + body, err := os.ReadFile(filepath.Join(source, "input")) + require.NoError(t, err) + assert.Equal(t, "fixture", string(body)) + require.NoError(t, p.Preflight(t.Context()), "subsequent jobs remain usable") + }) + } +} +func TestSandboxActivationRefusesMissingIsolation(t *testing.T) { + p, err := NewSubprocessParser(time.Second) + require.NoError(t, err) + cmd := exec.Command("true") + if !parserFDBootstrapAvailable { + require.ErrorIs(t, configureParserNamespace(cmd), ErrSandboxUnavailable) + require.ErrorIs(t, p.Preflight(t.Context()), ErrSandboxUnavailable) + return + } + require.NoError(t, configureParserNamespace(cmd)) + if err = cmd.Run(); err == nil { + t.Skip("host supports namespaces") + } + require.ErrorIs(t, p.Preflight(t.Context()), ErrSandboxUnavailable) +} + +func TestSandboxProviderJSONAndSQLite(t *testing.T) { + p, err := NewSubprocessParser(20 * time.Second) + require.NoError(t, err) + if err = p.Preflight(t.Context()); err != nil { + if os.Getenv("RAW_SANDBOX_REQUIRED") == "1" { + t.Fatal(err) + } + t.Skip("kernel isolation unavailable") + } + fixtures := []struct { + name string + provider parser.AgentType + path, key string + body []byte + ids []string + }{ + {"json", parser.AgentClaude, "project/session-a.jsonl", "/canonical/project/session-a.jsonl", []byte(`{"type":"user","timestamp":"2026-08-13T12:00:00Z","uuid":"u1","sessionId":"session-a","message":{"content":"hello"},"cwd":"/work/project"}` + "\n" + `{"type":"assistant","timestamp":"2026-08-13T12:00:01Z","uuid":"a1","parentUuid":"u1","sessionId":"session-a","message":{"content":"hi"}}` + "\n"), []string{"session-a"}}, + {"sqlite", parser.AgentForge, parser.ForgeDBFilename, parser.ForgeDBFilename, forgeSnapshotFixture(t, map[string]string{"conv-001": "2026-05-02 09:58:15", "conv-002": "2026-05-03 09:58:15"}), []string{"forge:conv-001", "forge:conv-002"}}, + } + for _, f := range fixtures { + t.Run(f.name, func(t *testing.T) { + ref := objectRefForBytes(t, f.body) + m := parserTestManifest(t, f.provider, f.key, []rawsync.Entry{{Path: f.path, Type: "file", Length: int64(len(f.body)), Objects: []rawsync.ObjectRef{ref}}}) + tree, err := (Materializer{Store: &materializerStore{objects: map[rawsync.ObjectRef][]byte{ref: f.body}}, BaseDir: t.TempDir(), MaxTotalBytes: 1 << 20}).Materialize(t.Context(), m) + require.NoError(t, err) + defer func() { require.NoError(t, tree.Cleanup()) }() + got, err := p.Parse(t.Context(), m, tree) + require.NoError(t, err) + require.Len(t, got.Outcome.Results, len(f.ids)) + assert.True(t, got.Outcome.ResultSetComplete) + assert.Empty(t, got.Outcome.SourceErrors) + for i, id := range f.ids { + assert.Equal(t, id, got.Outcome.Results[i].Result.Session.ID) + require.Len(t, got.Outcome.Results[i].Result.Messages, 2) + } + plain, err := NewProviderParser(parser.ProviderFactories(), "hosted") + require.NoError(t, err) + expected, err := plain.Parse(t.Context(), m, tree) + require.NoError(t, err) + assert.Equal(t, expected, got, "kernel boundary preserves complete normalized provider semantics") + }) + } +} + +func TestSandboxClosesDeliberatelyInheritedDescriptor(t *testing.T) { + if !parserFDBootstrapAvailable { + t.Skip("cgo pre-runtime FD bootstrap unavailable") + } + path := filepath.Join(t.TempDir(), "outside") + require.NoError(t, os.WriteFile(path, []byte("outside descriptor sentinel"), 0400)) + // unix.Open without O_CLOEXEC is deliberate: os.Open would hide the bug. + fd, err := unix.Open(path, unix.O_RDONLY, 0) + require.NoError(t, err) + defer unix.Close(fd) + pair, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0) + require.NoError(t, err) + defer unix.Close(pair[0]) + defer unix.Close(pair[1]) + require.NoError(t, unix.SetNonblock(pair[1], true)) + flags, err := unix.FcntlInt(uintptr(fd), unix.F_GETFD, 0) + require.NoError(t, err) + require.Zero(t, flags&unix.FD_CLOEXEC) + control := exec.Command(os.Args[0], "--sandbox-fd-test", strconv.Itoa(fd)) + control.Env = []string{"GOMAXPROCS=2"} + err = control.Run() + var exit *exec.ExitError + require.ErrorAs(t, err, &exit) + assert.Equal(t, 80, exit.ExitCode(), "control must actually inherit the sentinel") + cmd := exec.Command(os.Args[0], "--sandbox-fd-test", strconv.Itoa(fd), strconv.Itoa(pair[0])) + cmd.Env = []string{"GOMAXPROCS=2", "AGENTSVIEW_RAW_PARSER_CHILD=1"} + socketControl := exec.Command(os.Args[0], "--sandbox-fd-test", "-1", strconv.Itoa(pair[0])) + socketControl.Env = []string{"GOMAXPROCS=2"} + err = socketControl.Run() + require.ErrorAs(t, err, &exit) + assert.Equal(t, 80, exit.ExitCode(), "control must inherit the socket") + received := make([]byte, 64) + n, err := unix.Read(pair[1], received) + require.NoError(t, err) + assert.Equal(t, "outside socket sentinel", string(received[:n])) + _, err = unix.Seek(fd, 0, 0) + require.NoError(t, err) + output, err := cmd.Output() + require.NoError(t, err) + assert.Equal(t, "closed", string(output)) + data := make([]byte, 64) + n, err = unix.Read(fd, data) + require.NoError(t, err) + assert.Equal(t, "outside descriptor sentinel", string(data[:n]), "parent descriptor remains usable") + _, err = unix.Read(pair[1], data) + assert.ErrorIs(t, err, unix.EAGAIN, "child cannot use inherited socket") + require.False(t, parserFDBootstrapCompleted(), "ordinary parent must retain its descriptors") +} + +func TestSandboxSeccompSynchronizesThreads(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "--sandbox-kernel-test", "seccomp", "unused", "unused", "unused") + cmd.Env = []string{"GOMAXPROCS=2", "AGENTSVIEW_RAW_PARSER_CHILD=1"} + out, err := runParserProcess(ctx, cancel, cmd, nil, 4096, parserErrorLimit) + require.NoError(t, err) + var checks map[string]bool + require.NoError(t, json.Unmarshal(out, &checks)) + require.GreaterOrEqual(t, len(checks), 10) + for name, value := range checks { + assert.True(t, value, name) + } +} diff --git a/internal/rawderive/subprocess_presence_test.go b/internal/rawderive/subprocess_presence_test.go new file mode 100644 index 0000000000..41b58f0590 --- /dev/null +++ b/internal/rawderive/subprocess_presence_test.go @@ -0,0 +1,90 @@ +package rawderive + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" +) + +func TestSandboxGeminiWirePreservesTokenCoverage(t *testing.T) { + for _, tc := range []struct { + name, usage string + context, output bool + }{ + {"missing", "", false, false}, {"output_only", `,"tokens":{"output":5}`, false, true}, {"explicit_zero", `,"tokens":{"input":0,"output":0}`, true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "tmp", "project", "chats", "session-coverage.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + body := fmt.Sprintf(`{"sessionId":"coverage","startTime":"2026-09-01T12:00:00Z","lastUpdated":"2026-09-01T12:00:01Z","messages":[{"id":"m","type":"gemini","timestamp":"2026-09-01T12:00:01Z","content":"answer"%s}]}`, tc.usage) + require.NoError(t, os.WriteFile(path, []byte(body), 0400)) + factory, ok := parser.ProviderFactoryByType(parser.AgentGemini) + require.True(t, ok) + provider := factory.NewProvider(parser.ProviderConfig{Roots: []string{root}, Machine: "hosted"}) + outcome, err := provider.Parse(t.Context(), parser.ParseRequest{Source: parser.SourceRef{Provider: parser.AgentGemini, DisplayPath: path}}) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + encoded, err := encodeParserOutcome(ParsedManifest{Outcome: outcome}) + require.NoError(t, err) + wire, err := decodeParserOutcome(encoded) + require.NoError(t, err) + parsed := wire.Outcome.Results[0].Result + candidate, err := ingest.PrepareCandidate(t.Context(), parsed, ingest.ContentOptions{}) + require.NoError(t, err) + prepared, err := ingest.Finalize(t.Context(), candidate, ingest.ContentOptions{}) + require.NoError(t, err) + require.Len(t, prepared.Messages, 1) + assert.Equal(t, tc.context, prepared.Messages[0].HasContextTokens) + assert.Equal(t, tc.output, prepared.Messages[0].HasOutputTokens) + assert.Equal(t, tc.context, prepared.Session.HasPeakContextTokens) + assert.Equal(t, tc.output, prepared.Session.HasTotalOutputTokens) + assert.Equal(t, outcome, wire.Outcome, "transport preserves provider private state as well as exported fields") + }) + } +} + +func TestSandboxWirePreservesAuthoritativeAndLegacyPresence(t *testing.T) { + for _, tc := range []struct { + name string + known, flags, nonzero, want bool + }{ + {"authoritative_false", true, false, true, false}, {"explicit_zero", true, true, false, true}, {"missing", true, false, false, false}, {"legacy_nonzero", false, false, true, true}, {"legacy_zero_keys", false, false, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + message := parser.ParsedMessage{HasContextTokens: tc.flags, HasOutputTokens: tc.flags} + session := parser.ParsedSession{HasTotalOutputTokens: tc.flags, HasPeakContextTokens: tc.flags} + if tc.nonzero { + message.ContextTokens = 11 + message.OutputTokens = 7 + session.TotalOutputTokens = 7 + session.PeakContextTokens = 11 + } + if tc.name != "missing" { + message.TokenUsage = []byte(`{"input_tokens":0,"output_tokens":0}`) + } + message.RestoreTokenPresenceKnown(tc.known) + session.RestoreAggregateTokenPresenceKnown(tc.known) + data, err := encodeParserOutcome(ParsedManifest{Outcome: parser.ParseOutcome{Results: []parser.ParseResultOutcome{{Result: parser.ParseResult{Session: session, Messages: []parser.ParsedMessage{message}}}}}}) + require.NoError(t, err) + wire, err := decodeParserOutcome(data) + require.NoError(t, err) + result := wire.Outcome.Results[0].Result + context, output := result.Messages[0].TokenPresence() + assert.Equal(t, tc.want, context) + assert.Equal(t, tc.want, output) + total, peak := result.Session.AggregateTokenPresence() + wantSession := tc.want && tc.name != "legacy_zero_keys" + assert.Equal(t, wantSession, total) + assert.Equal(t, wantSession, peak) + assert.Equal(t, tc.known, result.Messages[0].TokenPresenceKnown()) + assert.Equal(t, tc.known, result.Session.AggregateTokenPresenceKnown()) + }) + } +} diff --git a/internal/rawderive/subprocess_test.go b/internal/rawderive/subprocess_test.go new file mode 100644 index 0000000000..bd381e70c8 --- /dev/null +++ b/internal/rawderive/subprocess_test.go @@ -0,0 +1,104 @@ +package rawderive + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/rawsync" +) + +func TestSandboxWirePreservesPartialOutcome(t *testing.T) { + in := ParsedManifest{Outcome: parser.ParseOutcome{ForceReplace: true, ResultSetComplete: false, ExcludedSessionIDs: []string{"excluded"}, SourceErrors: []parser.SourceError{{SessionID: "failed", SourceKey: "source", Retryable: true, Err: errors.New("private source contents")}}, Results: []parser.ParseResultOutcome{{DataVersion: parser.DataVersionNeedsRetry, RetryReason: "retry", Result: parser.ParseResult{Session: parser.ParsedSession{ID: "good"}}}}}} + in.Outcome.Results[0].Result.Messages = []parser.ParsedMessage{{ToolCalls: []parser.ParsedToolCall{{ResultEvents: []parser.ParsedToolResultEvent{{RawContentDigest: []byte{1, 2, 3}, Content: "result"}}}}}} + encoded, err := encodeParserOutcome(in) + require.NoError(t, err) + assert.NotContains(t, string(encoded), "private source contents") + out, err := decodeParserOutcome(encoded) + require.NoError(t, err) + assert.Equal(t, []string{"excluded"}, out.Outcome.ExcludedSessionIDs) + assert.Equal(t, []byte{1, 2, 3}, out.Outcome.Results[0].Result.Messages[0].ToolCalls[0].ResultEvents[0].RawContentDigest) + assert.True(t, out.Outcome.ForceReplace) + assert.False(t, out.Outcome.ResultSetComplete) + require.Len(t, out.Outcome.SourceErrors, 1) + assert.True(t, out.Outcome.SourceErrors[0].Retryable) + assert.Equal(t, "failed", out.Outcome.SourceErrors[0].SessionID) + require.Error(t, out.Outcome.SourceErrors[0].Err) + assert.Equal(t, parser.DataVersionNeedsRetry, out.Outcome.Results[0].DataVersion) + _, err = decodeParserOutcome(append(encoded, []byte(" {}")...)) + assert.Error(t, err) +} + +func TestParserProcessHarness(t *testing.T) { + if os.Getenv("RAW_TEST_CHILD") != "1" { + return + } + switch os.Args[len(os.Args)-1] { + case "blocked": + os.Stdout.WriteString("READY\n") + for { + time.Sleep(time.Hour) + } + case "stdout": + os.Stdout.WriteString("READY\n") + for { + os.Stdout.Write(bytes.Repeat([]byte("x"), 4096)) + } + case "stderr": + os.Stdout.WriteString("READY\n") + for { + os.Stderr.Write(bytes.Repeat([]byte("x"), 4096)) + } + } + os.Exit(0) +} +func TestParserProcessKillsAndReaps(t *testing.T) { + for _, mode := range []string{"blocked", "stdout", "stderr"} { + t.Run(mode, func(t *testing.T) { + timeout := 3 * time.Second + if mode == "blocked" { + timeout = 300 * time.Millisecond + } + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestParserProcessHarness$", "--", mode) + cmd.Env = []string{"RAW_TEST_CHILD=1"} + start := time.Now() + _, err := runParserProcess(ctx, cancel, cmd, []byte("{}"), 1024, 1024) + if mode == "blocked" { + require.ErrorIs(t, err, context.DeadlineExceeded) + } else { + require.ErrorIs(t, err, errParserProtocol) + } + require.NotNil(t, cmd.ProcessState) + assert.Less(t, time.Since(start), 3*time.Second) + assert.NotContains(t, err.Error(), strings.Repeat("x", 16)) + }) + } +} + +func TestSandboxProbeRequiresSourceSentinel(t *testing.T) { + root := t.TempDir() + assert.Error(t, verifyParserProbe(root)) + require.NoError(t, os.WriteFile(filepath.Join(root, parserProbeFile), []byte("wrong source"), 0400)) + assert.Error(t, verifyParserProbe(root)) + require.NoError(t, os.Remove(filepath.Join(root, parserProbeFile))) + require.NoError(t, os.WriteFile(filepath.Join(root, parserProbeFile), []byte(parserProbeContent), 0400)) + require.NoError(t, verifyParserProbe(root)) +} + +func TestSandboxTombstoneRequiresCanonicalValidation(t *testing.T) { + p, err := NewSubprocessParser(time.Second) + require.NoError(t, err) + _, err = p.Parse(t.Context(), rawsync.CanonicalManifest{Manifest: rawsync.Manifest{Kind: rawsync.ManifestTombstone}}, nil) + require.ErrorIs(t, err, rawsync.ErrInvalid) +} diff --git a/internal/rawderive/subprocess_unsupported.go b/internal/rawderive/subprocess_unsupported.go new file mode 100644 index 0000000000..6325fa4345 --- /dev/null +++ b/internal/rawderive/subprocess_unsupported.go @@ -0,0 +1,10 @@ +//go:build !linux || (!amd64 && !arm64) + +package rawderive + +import "os/exec" + +func configureParserNamespace(*exec.Cmd) error { return ErrSandboxUnavailable } +func isolateParser(string, string) error { return ErrSandboxUnavailable } + +func parserFDBootstrapCompleted() bool { return false } diff --git a/internal/rawderive/worker.go b/internal/rawderive/worker.go index 7abd1c4c24..5eec26db17 100644 --- a/internal/rawderive/worker.go +++ b/internal/rawderive/worker.go @@ -94,7 +94,9 @@ type SourceParser interface { } // ProjectionSink atomically applies a normalized outcome and completes the -// supplied fenced lease. Implementations must change neither projection nor +// supplied fenced lease. A partial commit returns CommittedProjectionOutcome +// after scheduling retry or terminal failure in that same transaction. +// Implementations must change neither projection nor // job state when the manifest is no longer the current source head. type ProjectionSink interface { Project(context.Context, JobLease, rawsync.CanonicalManifest, ParsedManifest) error @@ -167,11 +169,8 @@ func NewWorker(config WorkerConfig) (*Worker, error) { config.HeartbeatInterval >= config.LeaseDuration { return nil, fmt.Errorf("%w: raw parse worker lease timing is invalid", rawsync.ErrInvalid) } - if config.RetryBase <= 0 || config.RetryMax < config.RetryBase { - return nil, fmt.Errorf("%w: raw parse worker retry timing is invalid", rawsync.ErrInvalid) - } - if config.MaxAttempts <= 0 { - return nil, fmt.Errorf("%w: raw parse worker max attempts must be positive", rawsync.ErrInvalid) + if err := (RetryPolicy{Base: config.RetryBase, Maximum: config.RetryMax, MaxAttempts: config.MaxAttempts}).Validate(); err != nil { + return nil, err } if config.AttemptTimeout <= 0 { return nil, fmt.Errorf("%w: raw parse worker attempt timeout must be positive", rawsync.ErrInvalid) @@ -248,6 +247,16 @@ func (w *Worker) processLease(parent context.Context, lease JobLease) leaseResul workCause := context.Cause(workCtx) cancelWork(nil) cancelTimeout() + if committed, ok := errors.AsType[CommittedProjectionOutcome](operationErr); ok { + switch committed { + case ProjectionRetrying: + result.retried = 1 + return result + case ProjectionFailed: + result.failed = 1 + return result + } + } if operationErr == nil { // The pipeline committed its outcome; a heartbeat failure after // that point cannot un-commit it. @@ -300,7 +309,8 @@ func (w *Worker) processLease(parent context.Context, lease JobLease) leaseResul if parent.Err() != nil { return result } - if lease.Attempt >= w.MaxAttempts { + decision := (RetryPolicy{Base: w.RetryBase, Maximum: w.RetryMax, MaxAttempts: w.MaxAttempts}).Decide(lease.Attempt) + if decision.Failed { failureErr := w.Queue.FailRawParseJob( parent, lease, stage, jobErrorDiagnostic(stage, operationErr), ) @@ -318,7 +328,7 @@ func (w *Worker) processLease(parent context.Context, lease JobLease) leaseResul result.failed = 1 return result } - retryAt := w.now().Add(retryDelay(lease.Attempt, w.RetryBase, w.RetryMax)) + retryAt := w.now().Add(decision.Delay) retryErr := w.Queue.RetryRawParseJob( parent, lease, retryAt, stage, jobErrorDiagnostic(stage, operationErr), ) diff --git a/internal/rawtest/fixtures.go b/internal/rawtest/fixtures.go new file mode 100644 index 0000000000..a41ec8867f --- /dev/null +++ b/internal/rawtest/fixtures.go @@ -0,0 +1,124 @@ +package rawtest + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" +) + +const ClaudeID = "captured-session" +const ZCodeID = "zcode:captured-session" +const ZCodeBillableID = "zcode:billable-only" + +// Claude includes a persisted companion, a failed tool, thinking, explicit zero +// usage and absent usage. The synthetic credential-shaped text exercises findings. +func Claude(t *testing.T, root string) string { + t.Helper() + path := filepath.Join(root, "project", ClaudeID+".jsonl") + companion := filepath.Join(root, "project", ClaudeID, "tool-results", "result.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(companion), 0700)) + require.NoError(t, os.WriteFile(companion, []byte("Error: synthetic build failure\n"), 0600)) + notice, _ := json.Marshal("\nFull output saved to: " + companion + "\n") + quotedPath, _ := json.Marshal(companion) + // Derive a credential-shaped scanner input from public fixture text; no + // credential is used or retained by the test. + digest := sha256.Sum256([]byte("hosted capture scanner fixture")) + secret, _ := json.Marshal("Inspect the build. Synthetic scanner fixture: ghp_" + hex.EncodeToString(digest[:])[:36]) + body := `{"type":"user","uuid":"u1","sessionId":"` + ClaudeID + `","timestamp":"2026-07-06T12:00:00Z","cwd":"/workspace/project","message":{"content":` + string(secret) + `}} +{"type":"assistant","uuid":"a1","parentUuid":"u1","timestamp":"2026-07-06T12:00:01Z","message":{"id":"response-1","model":"claude-sonnet-4-6","content":[{"type":"thinking","thinking":"Check the build first."},{"type":"tool_use","id":"call-1","name":"Bash","input":{"command":"make check"}}],"usage":{"input_tokens":100,"output_tokens":7,"cache_creation_input_tokens":5,"cache_read_input_tokens":11}}} +{"type":"user","uuid":"u2","parentUuid":"a1","timestamp":"2026-07-06T12:00:02Z","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":` + string(notice) + `}]},"toolUseResult":{"persistedOutputPath":` + string(quotedPath) + `}} +{"type":"assistant","uuid":"a2","parentUuid":"u2","timestamp":"2026-07-06T12:00:03Z","message":{"model":"claude-sonnet-4-6","content":"The build failed.","usage":{"input_tokens":0,"output_tokens":0}}} +` + require.NoError(t, os.WriteFile(path, []byte(body), 0600)) + return path +} +func AppendClaude(t *testing.T, path string) { + t.Helper() + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) + require.NoError(t, err) + _, err = f.WriteString(`{"type":"user","uuid":"u3","parentUuid":"a2","timestamp":"2026-07-06T12:01:00Z","message":{"content":"Keep the failure visible."}} +{"type":"assistant","uuid":"a3","parentUuid":"u3","timestamp":"2026-07-06T12:01:01Z","message":{"model":"claude-sonnet-4-6","content":"Recorded the failure.","usage":{"input_tokens":120,"output_tokens":9}}} +`) + require.NoError(t, err) + require.NoError(t, f.Close()) +} + +// ZCode is a live WAL-backed provider database, including billable usage with no +// transcript. Capture must create a safe source snapshot before upload. +func ZCode(t *testing.T, root string) *sql.DB { + t.Helper() + dir := filepath.Join(root, "db") + require.NoError(t, os.MkdirAll(dir, 0700)) + source, err := sql.Open("sqlite3", filepath.Join(dir, parser.ZCodeDBName)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + _, err = source.Exec(`PRAGMA journal_mode=WAL; +CREATE TABLE session(id TEXT PRIMARY KEY,project_id TEXT,workspace_id TEXT,directory TEXT,title TEXT,time_created INTEGER,time_updated INTEGER); +CREATE TABLE message(id TEXT PRIMARY KEY,session_id TEXT,time_created TEXT,data TEXT); +CREATE TABLE part(id TEXT PRIMARY KEY,message_id TEXT,session_id TEXT,time_created TEXT,data TEXT); +CREATE TABLE model_usage(session_id TEXT,turn_id TEXT,provider_id TEXT,model_id TEXT,status TEXT,input_tokens INTEGER,output_tokens INTEGER,reasoning_tokens INTEGER,cache_creation_input_tokens INTEGER,cache_read_input_tokens INTEGER,computed_total_tokens INTEGER,started_at INTEGER,completed_at INTEGER,duration_ms INTEGER,tool_call_count INTEGER); +INSERT INTO session VALUES('captured-session',NULL,NULL,'/workspace/project','Captured database session',1783339200000,1783339260000),('billable-only',NULL,NULL,'/workspace/project','Billable without transcript',1783339200000,1783339260000); +INSERT INTO message VALUES('u1','captured-session','2026-07-06T12:00:00Z','{"role":"user"}'),('a1','captured-session','2026-07-06T12:00:01Z','{"role":"assistant","model":{"modelID":"claude-sonnet-4-6"}}'); +INSERT INTO part VALUES('p1','u1','captured-session','2026-07-06T12:00:00Z','{"type":"text","text":"Inspect the build."}'),('p2','a1','captured-session','2026-07-06T12:00:01Z','{"type":"text","text":"Build inspected."}'); +INSERT INTO model_usage VALUES('captured-session','turn-1','synthetic','claude-sonnet-4-6','done',100,7,3,5,11,126,1783339201000,1783339202000,1000,0),('billable-only','turn-2','synthetic','claude-sonnet-4-6','done',200,17,4,6,12,239,1783339201000,1783339202000,1000,0);`) + require.NoError(t, err) + return source +} + +const CodexChildID = "22222222-2222-4222-8222-222222222222" +const CodexParentID = "11111111-1111-4111-8111-111111111111" + +func CodexFork(t *testing.T, root string) string { + t.Helper() + dir := filepath.Join(root, "2026", "07", "06") + require.NoError(t, os.MkdirAll(dir, 0700)) + path := filepath.Join(dir, "rollout-2026-07-06T12-00-00-"+CodexChildID+".jsonl") + body := `{"type":"session_meta","timestamp":"2026-07-06T12:00:00Z","payload":{"id":"` + CodexChildID + `","forked_from_id":"` + CodexParentID + `","cwd":"/workspace/project","originator":"codex_cli_rs"}} +{"type":"turn_context","timestamp":"2026-07-06T12:00:01Z","payload":{"model":"gpt-5.4","turn_id":"child-turn"}} +{"type":"response_item","timestamp":"2026-07-06T12:00:01Z","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Inspect the fork."}]}} +{"type":"response_item","timestamp":"2026-07-06T12:00:02Z","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Fork inspected."}]}} +` + require.NoError(t, os.WriteFile(path, []byte(body), 0600)) + return path +} +func CodexParent(t *testing.T, root string) { + t.Helper() + path := filepath.Join(root, "2026", "07", "06", "rollout-2026-07-06T11-00-00-"+CodexParentID+".jsonl") + require.NoError(t, os.WriteFile(path, []byte(`{"type":"session_meta","timestamp":"2026-07-06T11:00:00Z","payload":{"id":"`+CodexParentID+`","cwd":"/workspace/project","originator":"codex_cli_rs"}} +`), 0600)) +} + +const CodexToolsID = "33333333-3333-4333-8333-333333333333" + +func CodexTools(t *testing.T, root string) { + t.Helper() + dir := filepath.Join(root, "2026", "07", "06") + require.NoError(t, os.MkdirAll(dir, 0700)) + body := `{"type":"session_meta","timestamp":"2026-07-06T12:00:00Z","payload":{"id":"` + CodexToolsID + `","cwd":"/workspace/project","originator":"codex_cli_rs"}} +{"type":"turn_context","timestamp":"2026-07-06T12:00:01Z","payload":{"model":"gpt-5.4","turn_id":"tool-turn"}} +{"type":"response_item","timestamp":"2026-07-06T12:00:01Z","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Run the build."}]}} +{"type":"response_item","timestamp":"2026-07-06T12:00:02Z","payload":{"type":"function_call","name":"exec_command","call_id":"build-1","arguments":"{\"cmd\":\"make check\"}"}} +{"type":"response_item","timestamp":"2026-07-06T12:00:03Z","payload":{"type":"function_call_output","call_id":"build-1","output":"Process exited with code 1\nError: synthetic failure"}} +{"type":"response_item","timestamp":"2026-07-06T12:00:04Z","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Build failed."}]}} +{"type":"event_msg","timestamp":"2026-07-06T12:00:05Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":120,"cached_input_tokens":20,"output_tokens":9,"total_tokens":129},"last_token_usage":{"input_tokens":120,"cached_input_tokens":20,"output_tokens":9,"total_tokens":129}}}} +` + require.NoError(t, os.WriteFile(filepath.Join(dir, "rollout-2026-07-06T12-00-00-"+CodexToolsID+".jsonl"), []byte(body), 0600)) +} + +const ClaudeChildID = "agent-captured-child" + +func ClaudeChild(t *testing.T, root string) { + t.Helper() + path := filepath.Join(root, "project", ClaudeID, "subagents", ClaudeChildID+".jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte(`{"type":"user","uuid":"child-u1","isSidechain":true,"timestamp":"2026-07-06T12:00:01Z","cwd":"/workspace/project","message":{"content":"Inspect the child task."}} +{"type":"assistant","uuid":"child-a1","parentUuid":"child-u1","isSidechain":true,"timestamp":"2026-07-06T12:00:02Z","message":{"content":"Child task inspected."}} +`), 0600)) +} diff --git a/internal/rawtest/parity.go b/internal/rawtest/parity.go new file mode 100644 index 0000000000..8ee296603c --- /dev/null +++ b/internal/rawtest/parity.go @@ -0,0 +1,158 @@ +// Package rawtest holds synthetic capture fixtures and stored-result assertions +// shared by the PostgreSQL pipeline and executable runtime integration tests. +package rawtest + +import ( + "context" + "database/sql" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/money" + "go.kenn.io/agentsview/internal/parser" + agentsync "go.kenn.io/agentsview/internal/sync" +) + +func Oracle(t *testing.T, agent parser.AgentType, root string, policy config.ArchiveContent, images config.ToolResultImages) (*db.DB, *agentsync.Engine) { + t.Helper() + archive, err := db.OpenIsolated(filepath.Join(t.TempDir(), "oracle.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, archive.Close()) }) + archive.SetArchiveContent(policy) + engine := agentsync.NewEngine(archive, agentsync.EngineConfig{AgentDirs: map[parser.AgentType][]string{agent: {root}}, Machine: "capture-device", Ephemeral: true, DisableFilesystemProjectDiscovery: true, ToolResultImages: images}) + t.Cleanup(engine.Close) + return archive, engine +} + +// EqualStored catches lost content, tool pairing, usage presence, signals, +// findings and public relationship mappings. Expected rows come from the local +// engine, never the hosted preparation/projection API. +func EqualStored(t *testing.T, ctx context.Context, want *db.DB, got db.Store, ids ...string) { + t.Helper() + for _, id := range ids { + a, err := want.GetSessionFull(ctx, id) + require.NoError(t, err) + require.NotNil(t, a, id) + b, err := got.GetSessionFull(ctx, id) + require.NoError(t, err) + require.NotNil(t, b, id) + assert.Equal(t, normalizeSession(*a), normalizeSession(*b), "session %s", id) + am, err := want.GetAllMessages(ctx, id) + require.NoError(t, err) + bm, err := got.GetAllMessages(ctx, id) + require.NoError(t, err) + assert.Equal(t, normalizeMessages(t, am), normalizeMessages(t, bm), "messages %s", id) + au, err := want.GetSessionUsage(ctx, id, true) + require.NoError(t, err) + bu, err := got.GetSessionUsage(ctx, id, true) + require.NoError(t, err) + assert.Equal(t, au, bu, "usage %s", id) + } + af, err := want.ListSecretFindings(ctx, db.SecretFindingFilter{Limit: 1000}) + require.NoError(t, err) + bf, err := got.ListSecretFindings(ctx, db.SecretFindingFilter{Limit: 1000}) + require.NoError(t, err) + assert.Equal(t, af, bf, "complete findings and coordinates") +} + +func normalizeSession(s db.Session) db.Session { + // Machine and file coordinates identify the custody transport, not content. + s.Machine = "" + s.FilePath = nil + s.FileSize = nil + s.FileMtime = nil + s.FileInode = nil + s.FileDevice = nil + s.FileHash = nil + // These fields are archive-local import bookkeeping or publication clocks. + s.NextOrdinal = 0 + s.LastEntryUUID = nil + s.ClaudeLinearParse = nil + s.LastWriteIncremental = false + // PostgreSQL returns the effective DisplayName but does not expose the + // SQLite-only raw session_name read field. DisplayName remains compared. + s.SessionName = nil + // The hosted public adapter deliberately clears the internal parser parent + // cache; the resolved public ParentSessionID(s) remain fully compared. + s.ParserParentSessionID = nil + s.CreatedAt = "" + s.LocalModifiedAt = nil + s.TranscriptRevision = nil + return s +} + +func normalizeMessages(t *testing.T, messages []db.Message) []db.Message { + // Empty SQL result sets differ only in nil versus allocated slice transport. + if len(messages) == 0 { + return nil + } + for i := range messages { + m := &messages[i] + m.ID = 0 + // PostgreSQL JSONB changes key order/whitespace, never the usage values. + if len(m.TokenUsage) > 0 { + var v any + require.NoError(t, json.Unmarshal(m.TokenUsage, &v)) + b, err := json.Marshal(v) + require.NoError(t, err) + m.TokenUsage = b + } + m.Timestamp = normalizeTime(m.Timestamp) + for j := range m.ToolCalls { + c := &m.ToolCalls[j] + c.MessageID = 0 + for k := range c.ResultEvents { + e := &c.ResultEvents[k] + e.Timestamp = normalizeTime(e.Timestamp) + // SQLite-only late-result reconciliation state is deliberately not mirrored. + e.RawContentDigest = nil + e.SummaryParticipates = nil + } + } + } + return messages +} +func normalizeTime(s string) string { + if v, err := time.Parse(time.RFC3339Nano, s); err == nil { + return v.UTC().Format(time.RFC3339Nano) + } + return s +} + +// EqualUsageEvents preserves the complete normalized accounting rows, including +// nullable money/message links, provider/source keys and deduplication identity. +func EqualUsageEvents(t *testing.T, ctx context.Context, oracle *db.DB, pg *sql.DB, publicID, physicalID string) { + t.Helper() + want, err := oracle.GetUsageEvents(ctx, publicID) + require.NoError(t, err) + rows, err := pg.QueryContext(ctx, `SELECT message_ordinal,source,model,provider_id,input_tokens,output_tokens,cache_creation_input_tokens,cache_read_input_tokens,reasoning_tokens,cost_microdollars,cost_status,cost_source,occurred_at,dedup_key FROM usage_events WHERE session_id=$1 ORDER BY occurred_at NULLS FIRST,id`, physicalID) + require.NoError(t, err) + defer rows.Close() + var got []db.UsageEvent + for rows.Next() { + var e db.UsageEvent + var cost *int64 + var occurred *string + require.NoError(t, rows.Scan(&e.MessageOrdinal, &e.Source, &e.Model, &e.ProviderID, &e.InputTokens, &e.OutputTokens, &e.CacheCreationInputTokens, &e.CacheReadInputTokens, &e.ReasoningTokens, &cost, &e.CostStatus, &e.CostSource, &occurred, &e.DedupKey)) + if cost != nil { + e.Cost = &money.Money{Microdollars: *cost} + } + if occurred != nil { + e.OccurredAt = normalizeTime(*occurred) + } + e.SessionID = publicID + got = append(got, e) + } + require.NoError(t, rows.Err()) + for i := range want { + want[i].ID = 0 + want[i].OccurredAt = normalizeTime(want[i].OccurredAt) + } + assert.Equal(t, want, got, "complete usage events for %s", publicID) +} diff --git a/internal/rawtest/parity_test.go b/internal/rawtest/parity_test.go new file mode 100644 index 0000000000..859e013cd4 --- /dev/null +++ b/internal/rawtest/parity_test.go @@ -0,0 +1,136 @@ +package rawtest + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +func TestCaptureOracleLiteralContent(t *testing.T) { + root := t.TempDir() + Claude(t, root) + archive, engine := Oracle(t, parser.AgentClaude, root, "", "") + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + session, err := archive.GetSessionFull(t.Context(), ClaudeID) + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, ClaudeID, session.SourceSessionID) + assert.True(t, session.HasToolCalls) + assert.True(t, session.HasContextData) + assert.Equal(t, 1, session.SecretLeakCount) + messages, err := archive.GetAllMessages(t.Context(), ClaudeID) + require.NoError(t, err) + require.Len(t, messages, 3) + require.Len(t, messages[1].ToolCalls, 1) + call := messages[1].ToolCalls[0] + assert.Equal(t, "Bash", call.ToolName) + assert.Equal(t, "Error: synthetic build failure\n", call.ResultContent) + assert.Empty(t, call.ResultEvents, "Claude persists its summary without Codex lifecycle events") + assert.Equal(t, "Check the build first.", messages[1].ThinkingText) + assert.Equal(t, 7, messages[1].OutputTokens) + assert.Equal(t, 116, messages[1].ContextTokens) + assert.True(t, messages[2].HasOutputTokens) + assert.True(t, messages[2].HasContextTokens) + assert.Zero(t, messages[2].OutputTokens) + assert.False(t, messages[0].HasOutputTokens) + assert.False(t, messages[0].HasContextTokens) + findings, err := archive.ListSecretFindings(t.Context(), db.SecretFindingFilter{}) + require.NoError(t, err) + require.Len(t, findings.Findings, 1) + finding := findings.Findings[0] + assert.Equal(t, "github-pat", finding.RuleName) + assert.Equal(t, "message", finding.LocationKind) + assert.Equal(t, 0, finding.MessageOrdinal) + assert.Nil(t, finding.CallIndex) + assert.Nil(t, finding.EventIndex) + assert.Equal(t, 46, finding.MatchStart) + assert.Equal(t, 86, finding.MatchEnd) + assert.Equal(t, 0, finding.MatchIndex) +} + +func TestCaptureForkPartialAndCompanionRecovery(t *testing.T) { + root := t.TempDir() + CodexFork(t, root) + provider, ok := parser.NewProvider(parser.AgentCodex, parser.ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := parser.DiscoverRawCaptureSources(t.Context(), provider) + require.NoError(t, err) + require.Len(t, sources.Sources, 1) + out, err := provider.Parse(t.Context(), parser.ParseRequest{Source: sources.Sources[0], ForceParse: true}) + require.NoError(t, err) + require.Len(t, out.Results, 1) + assert.Equal(t, parser.DataVersionNeedsRetry, out.Results[0].DataVersion) + assert.Len(t, out.Results[0].Result.Messages, 2) + CodexParent(t, root) + // A fresh provider avoids allowing an earlier unresolved-parent cache entry + // to stand in for a newly accepted immutable generation. + provider, ok = parser.NewProvider(parser.AgentCodex, parser.ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source, found, err := provider.FindSource(t.Context(), parser.FindSourceRequest{RawSessionID: CodexChildID}) + require.NoError(t, err) + require.True(t, found) + out, err = provider.Parse(t.Context(), parser.ParseRequest{Source: source, ForceParse: true}) + require.NoError(t, err) + require.Len(t, out.Results, 1) + assert.Equal(t, parser.DataVersionCurrent, out.Results[0].DataVersion) + plan, supported, err := parser.ResolveRawCapturePlan(t.Context(), provider, source) + require.NoError(t, err) + require.True(t, supported) + require.Len(t, plan.Entries, 2) + for _, entry := range plan.Entries { + _, err := os.Stat(filepath.Clean(entry.LocalPath)) + require.NoError(t, err) + } +} + +func TestCaptureOracleRelationshipsAndToolEvents(t *testing.T) { + t.Run("claude parent", func(t *testing.T) { + root := t.TempDir() + Claude(t, root) + ClaudeChild(t, root) + archive, engine := Oracle(t, parser.AgentClaude, root, "", "") + require.Equal(t, 2, engine.SyncAll(t.Context(), nil).Synced) + child, err := archive.GetSessionFull(t.Context(), ClaudeChildID) + require.NoError(t, err) + require.NotNil(t, child) + require.NotNil(t, child.ParentSessionID) + assert.Equal(t, ClaudeID, *child.ParentSessionID) + assert.Equal(t, "subagent", child.RelationshipType) + }) + t.Run("codex tool events", func(t *testing.T) { + root := t.TempDir() + CodexTools(t, root) + archive, engine := Oracle(t, parser.AgentCodex, root, "", "") + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + messages, err := archive.GetAllMessages(t.Context(), "codex:"+CodexToolsID) + require.NoError(t, err) + var events []db.ToolResultEvent + for _, message := range messages { + for _, call := range message.ToolCalls { + events = append(events, call.ResultEvents...) + } + } + require.Len(t, events, 1) + assert.Equal(t, "Process exited with code 1\nError: synthetic failure", events[0].Content) + }) +} + +func TestCaptureOracleBillableWithoutMessages(t *testing.T) { + root := t.TempDir() + ZCode(t, root) + archive, engine := Oracle(t, parser.AgentZCode, root, "", "") + require.Equal(t, 2, engine.SyncAll(t.Context(), nil).Synced) + messages, err := archive.GetAllMessages(t.Context(), ZCodeBillableID) + require.NoError(t, err) + assert.Empty(t, messages) + usage, err := archive.GetSessionUsage(t.Context(), ZCodeBillableID, true) + require.NoError(t, err) + require.NotNil(t, usage) + assert.Equal(t, 17, usage.TotalOutputTokens) + assert.True(t, usage.HasCost) +} diff --git a/internal/server/auth.go b/internal/server/auth.go index b2d4f0308e..0e9476669c 100644 --- a/internal/server/auth.go +++ b/internal/server/auth.go @@ -237,7 +237,13 @@ func (s *Server) isRawSyncOwnAuthPath(path string) bool { func (s *Server) authenticateRawSyncRequest( r *http.Request, -) (rawsync.AuthIdentity, error) { +) (identity rawsync.AuthIdentity, resultErr error) { + defer func() { + if resultErr == nil && s.rawSyncTenant != "" && identity.TenantID != s.rawSyncTenant { + identity = rawsync.AuthIdentity{} + resultErr = rawsync.ErrUnauthorized + } + }() secret, err := rawSyncBearer(r.Header.Get("Authorization")) if err != nil { return rawsync.AuthIdentity{}, err diff --git a/internal/server/huma_routes.go b/internal/server/huma_routes.go index a99c98c984..613aeb958e 100644 --- a/internal/server/huma_routes.go +++ b/internal/server/huma_routes.go @@ -45,13 +45,15 @@ type bytesOutput struct { } type apiErrorResponse struct { - Status int `json:"-"` - Code string `json:"code,omitempty"` - Message string `json:"error"` - CurrentManifestID string `json:"current_manifest_id,omitempty"` - CurrentReceipt string `json:"current_receipt,omitempty"` - CurrentGeneration int64 `json:"current_generation,omitzero"` - CurrentUploadOffset *int64 `json:"upload_offset,omitempty"` + Variants []string `json:"variants,omitempty"` + IdentityState string `json:"state,omitempty"` + Status int `json:"-"` + Code string `json:"code,omitempty"` + Message string `json:"error"` + CurrentManifestID string `json:"current_manifest_id,omitempty"` + CurrentReceipt string `json:"current_receipt,omitempty"` + CurrentGeneration int64 `json:"current_generation,omitzero"` + CurrentUploadOffset *int64 `json:"upload_offset,omitempty"` } func (e *apiErrorResponse) Error() string { @@ -518,6 +520,9 @@ func (s *Server) serializeArchiveWrite(work func() error) error { } func serverError(err error) error { + if handled := hostedIdentityHTTPError(err); handled != nil { + return handled + } if errors.Is(err, context.Canceled) { return nil } @@ -528,6 +533,9 @@ func serverError(err error) error { } func internalError(logPrefix string, err error) error { + if handled := hostedIdentityHTTPError(err); handled != nil { + return handled + } if errors.Is(err, context.Canceled) { return nil } @@ -598,3 +606,14 @@ func writeHumaJSON(ctx huma.Context, status int, value any) { func sjson(w io.Writer, value any) error { return json.MarshalEncode(jsontext.NewEncoder(w), value) } + +func hostedIdentityHTTPError(err error) error { + if identity, ok := errors.AsType[*db.SessionIdentityError](err); ok { + status := http.StatusNotFound + if identity.State == "ambiguous" { + status = http.StatusConflict + } + return &apiErrorResponse{Status: status, Message: identity.Error(), IdentityState: identity.State, Variants: identity.Variants} + } + return nil +} diff --git a/internal/server/huma_routes_pins.go b/internal/server/huma_routes_pins.go index a1af5a2e6c..65c85420fe 100644 --- a/internal/server/huma_routes_pins.go +++ b/internal/server/huma_routes_pins.go @@ -10,6 +10,7 @@ import ( func (s *Server) registerPinRoutes() { group := newRouteGroup(s.api, "/api/v1", "Pins") + s.deleteRoute(group, "/sessions/{id}/pin-references/{messageKey}", "Remove retained pin", s.humaRemovePinReference) s.get(group, "/pins", "List pins", s.humaListPins) s.get(group, "/sessions/{id}/pins", "List session pins", s.humaListSessionPins) s.post(group, "/sessions/{id}/messages/{messageId}/pin", "Pin message", s.humaPinMessage) @@ -95,3 +96,19 @@ func (s *Server) humaUnpinMessage( } return &noContentOutput{Status: http.StatusNoContent}, nil } + +type pinReferenceInput struct { + ID string `path:"id"` + MessageKey string `path:"messageKey"` +} + +func (s *Server) humaRemovePinReference(ctx context.Context, in *pinReferenceInput) (*noContentOutput, error) { + store, ok := s.db.(db.PinReferenceStore) + if !ok { + return nil, apiError(http.StatusNotImplemented, "pin references are not available") + } + if err := store.RemovePinReference(ctx, in.ID, in.MessageKey); err != nil { + return nil, internalError("remove pin reference", err) + } + return &noContentOutput{Status: http.StatusNoContent}, nil +} diff --git a/internal/server/huma_routes_raw_sync_test.go b/internal/server/huma_routes_raw_sync_test.go index 47a3e5b90b..c5168d79e4 100644 --- a/internal/server/huma_routes_raw_sync_test.go +++ b/internal/server/huma_routes_raw_sync_test.go @@ -801,3 +801,22 @@ func (s *rawSyncCustodyStub) CommitManifest( var _ RawSyncDeviceAuth = (*rawSyncAuthStub)(nil) var _ RawSyncCustody = (*rawSyncCustodyStub)(nil) + +func TestRawSyncRejectsOtherTenantBeforeCustody(t *testing.T) { + other := rawsync.AuthIdentity{TenantID: "other", DeviceID: "device"} + for _, path := range []string{"/api/v1/raw-sync/tokens", "/api/v1/raw-sync/objects/missing", "/api/v1/raw-sync/manifests", "/api/v1/raw-sync/uploads"} { + t.Run(path, func(t *testing.T) { + auth := &rawSyncAuthStub{authenticateToken: func(context.Context, string, rawsync.DeviceTokenScope) (rawsync.AuthIdentity, error) { + return other, nil + }, authenticateCredential: func(context.Context, string, string) (rawsync.AuthIdentity, error) { return other, nil }} + custody := &rawSyncCustodyStub{} + uploads := &rawSyncUploadsStub{} + srv := New(config.Config{Host: "127.0.0.1", Port: 8080, RequireAuth: true, AuthToken: "synthetic-server-token"}, nil, nil, WithRawSyncServices(auth, custody), WithRawSyncUploads(uploads), WithRawSyncTenant("configured")) + recorder := serveRawSyncJSON(t, srv, http.MethodPost, path, `{}`, "avdt_token", "device") + assert.Equal(t, http.StatusUnauthorized, recorder.Code) + assert.Zero(t, custody.missingCalls) + assert.Zero(t, custody.commitCalls) + assert.Zero(t, uploads.startCalls) + }) + } +} diff --git a/internal/server/huma_routes_sessions.go b/internal/server/huma_routes_sessions.go index fa54eb8605..a961673972 100644 --- a/internal/server/huma_routes_sessions.go +++ b/internal/server/huma_routes_sessions.go @@ -980,6 +980,17 @@ func (s *Server) humaWatchSession( if !ok { return } + if identity, ok := s.db.(db.SessionWatchStateStore); ok { + state, e := identity.GetSessionWatchState(in.ID) + if e != nil { + return + } + if state.State != db.SessionWatchResolved { + stream.SendJSON("session.identity", state) + return + } + stream.SendJSON("session.identity", state) + } stream.Send("session_updated", in.ID) if t, err := s.db.GetSessionTiming(streamCtx, in.ID); err != nil { log.Printf("session timing update: %v", err) @@ -1298,6 +1309,12 @@ func (s *Server) humaResumeSession( } prefix := string(session.Agent) + ":" rawID := strings.TrimPrefix(in.ID, prefix) + if provider, ok := s.db.(db.ProviderResumeIdentityStore); ok { + rawID, err = provider.GetProviderResumeID(ctx, in.ID) + if err != nil { + return nil, internalError("resume identity", err) + } + } if s.db.ReadOnly() && !req.CommandOnly { return nil, apiError(http.StatusNotImplemented, "session launch not available in remote mode") diff --git a/internal/server/server.go b/internal/server/server.go index 7c9414c87f..2de95c46ab 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -178,6 +178,7 @@ type Server struct { localCompactRunner LocalCompactRunner artifactExchangeRunner ArtifactExchangeRunner + rawSyncTenant string rawSyncDeviceAuth RawSyncDeviceAuth rawSyncCustody RawSyncCustody rawSyncSchemaOnly bool @@ -1576,3 +1577,6 @@ func logMiddleware(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } + +// WithRawSyncTenant binds every raw credential/token identity before route work. +func WithRawSyncTenant(tenant string) Option { return func(s *Server) { s.rawSyncTenant = tenant } } diff --git a/internal/service/direct.go b/internal/service/direct.go index f072e61cdb..49e8dea28f 100644 --- a/internal/service/direct.go +++ b/internal/service/direct.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json/v2" "errors" "fmt" "io" @@ -626,6 +627,24 @@ func (b *directBackend) Watch( if !ok { return } + if identity, ok := b.db.(db.SessionWatchStateStore); ok { + state, e := identity.GetSessionWatchState(id) + if e != nil { + return + } + payload, e := json.Marshal(state) + if e != nil { + return + } + select { + case out <- Event{Event: "session.identity", Data: string(payload)}: + case <-ctx.Done(): + return + } + if state.State != db.SessionWatchResolved { + return + } + } select { case out <- Event{Event: "session_updated", Data: id}: case <-ctx.Done(): diff --git a/internal/sessionwatch/watcher.go b/internal/sessionwatch/watcher.go index 304f498998..2ecb6789e9 100644 --- a/internal/sessionwatch/watcher.go +++ b/internal/sessionwatch/watcher.go @@ -7,6 +7,7 @@ import ( "context" "log" "os" + "slices" "sync/atomic" "time" @@ -89,6 +90,50 @@ func (w *Watcher) Events( ctx context.Context, sessionID string, ) <-chan struct{} { ch := make(chan struct{}) + if identity, ok := w.db.(db.SessionWatchStateStore); ok { + initial, err := identity.GetSessionWatchState(sessionID) + go func() { + defer close(ch) + if err != nil { + return + } + if initial.State != db.SessionWatchResolved { + select { + case ch <- struct{}{}: + case <-ctx.Done(): + } + return + } + ticker := time.NewTicker(pollInterval()) + defer ticker.Stop() + lastCount, lastVersion, _ := w.db.GetSessionVersion(sessionID) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + current, e := identity.GetSessionWatchState(sessionID) + if e != nil { + continue + } + transition := current.State != initial.State || current.PublicID != initial.PublicID || current.ContentRevision != initial.ContentRevision || !slices.Equal(current.Variants, initial.Variants) + count, version, ok := w.db.GetSessionVersion(sessionID) + if transition || ok && (count != lastCount || version != lastVersion) { + select { + case ch <- struct{}{}: + case <-ctx.Done(): + return + } + if transition { + return + } + lastCount, lastVersion = count, version + } + } + } + }() + return ch + } lastCount, lastDBVersion, _ := w.db.GetSessionVersion( sessionID, ) diff --git a/internal/signals/outcome.go b/internal/signals/outcome.go index 7a07b78025..473612ec8a 100644 --- a/internal/signals/outcome.go +++ b/internal/signals/outcome.go @@ -38,12 +38,17 @@ var giveUpPatterns = []string{ // ClassifyOutcome classifies a session's outcome based on its // metadata. Pure computation, no DB access. func ClassifyOutcome(in OutcomeInput) OutcomeResult { + return ClassifyOutcomeAt(in, time.Now()) +} + +// ClassifyOutcomeAt evaluates recency at an explicit observation time. +func ClassifyOutcomeAt(in OutcomeInput, now time.Time) OutcomeResult { if in.IsAutomated { return OutcomeResult{"unknown", "low", false} } if in.EndedWithRole == "assistant" && hasTerminalAPIErrorText(in.LastAssistantText) { - if isRecent(in.LastActivity) { + if isRecentAt(in.LastActivity, now) { return OutcomeResult{"unknown", "low", true} } return OutcomeResult{"errored", "medium", false} @@ -57,7 +62,7 @@ func ClassifyOutcome(in OutcomeInput) OutcomeResult { return OutcomeResult{"unknown", "low", false} } - if isRecent(in.LastActivity) { + if isRecentAt(in.LastActivity, now) { return OutcomeResult{"unknown", "low", true} } @@ -84,11 +89,11 @@ func ClassifyOutcome(in OutcomeInput) OutcomeResult { return OutcomeResult{"unknown", "low", false} } -func isRecent(t time.Time) bool { +func isRecentAt(t, now time.Time) bool { if t.IsZero() { return false } - return time.Since(t) < RecencyWindow + return now.Sub(t) < RecencyWindow } func hasGiveUpPattern(text string) bool { diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 62a322ef58..d3d4996679 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -24,12 +24,10 @@ import ( "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/export" + "go.kenn.io/agentsview/internal/ingest" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/pathutil" "go.kenn.io/agentsview/internal/secrets" - "go.kenn.io/agentsview/internal/signals" - "go.kenn.io/agentsview/internal/timeutil" - "go.kenn.io/agentsview/internal/usagefacts" ) const ( @@ -6245,12 +6243,7 @@ func isOpenCodeFormatAgent(agent parser.AgentType) bool { // or its S3 archive layout stay keyed to parser.AgentCodex alone: TraeX writes // no index file and has no S3 path convention. func isCodexFormatAgent(agent parser.AgentType) bool { - switch agent { - case parser.AgentCodex, parser.AgentTraeX: - return true - default: - return false - } + return ingest.IsCodexFormatAgent(agent) } func reconciliationWatchRoot( @@ -8853,7 +8846,7 @@ func claudeDiscoveredFileKey( // OpenClaude also stores this layout but parses linearly to one session per // file, so it stays outside this set. func isClaudeFormatAgent(agent parser.AgentType) bool { - return agent == parser.AgentClaude || agent == parser.AgentIcodemate + return ingest.IsClaudeFormatAgent(agent) } func isClaudeFormatTranscriptFile(file parser.DiscoveredFile) bool { @@ -8864,7 +8857,7 @@ func isClaudeFormatTranscriptFile(file parser.DiscoveredFile) bool { // path. For ICodeMate this distinguishes CLI transcripts from the agent's // OpenCode-storage family, which shares the same agent label. func isClaudeFormatTranscript(agent parser.AgentType, path string) bool { - return isClaudeFormatAgent(agent) && claudeSessionIDFromPath(path) != "" + return ingest.IsClaudeFormatTranscript(agent, path) } // claudeFormatArchiveSessionID maps a raw filename-derived session ID into @@ -15960,20 +15953,6 @@ func commandCodeEffectiveInfo(path string, info os.FileInfo) os.FileInfo { return fakeSnapshotInfo{fSize: size, fMtime: mtime} } -// computeFinalStreak counts trailing consecutive failures -// from the end of the tool call list. -func computeFinalStreak(calls []signals.ToolCallRow) int { - streak := 0 - for _, v := range slices.Backward(calls) { - if signals.IsFailure(v) { - streak++ - } else { - break - } - } - return streak -} - // RecomputeSignals recomputes signals for a single session // from existing DB data. Returns nil on success (including // when the session no longer exists). Returns an error when @@ -16767,8 +16746,8 @@ func (e *Engine) writeBatchWithOutcomeContext( if ctx.Err() != nil { return outcome } - s, msgs, verdict, prepErr := e.prepareSessionWriteContext( - ctx, pw, resolveWorktreeProject, + prepared, verdict, prepErr := e.prepareSessionNormalizedContext( + ctx, pw, resolveWorktreeProject, true, ) if prepErr != nil { return outcome @@ -16779,6 +16758,7 @@ func (e *Engine) writeBatchWithOutcomeContext( } continue } + s, msgs := prepared.Session, prepared.Messages // Detect stale parser version BEFORE UpsertSession // overwrites it. Existing message rows from an // older parser lack new metadata columns, and newly @@ -16825,8 +16805,8 @@ func (e *Engine) writeBatchWithOutcomeContext( pw, forceReplace, stale, revivingSourceMissing, ) - var update db.SessionSignalUpdate - var findings []db.SecretFinding + update := prepared.Signals + findings := prepared.Findings var werr error if replaceMessages && pw.staged != nil { // The staged sink owns this parse's tool-result rows, and only the @@ -16834,7 +16814,7 @@ func (e *Engine) writeBatchWithOutcomeContext( // recomputation is disabled. werr = e.writeStagedFullParse(ctx, s, msgs, pw) } else if replaceMessages && !e.disableSignalRecompute { - update, findings, werr = e.computeFullSignalsAndSecretsForStorage(s, msgs, nil) + update, werr = e.attachFullSignalState(s, msgs, update, nil) if werr != nil { log.Printf("compute full signals %s: %v", s.ID, werr) outcome.failedSessions++ @@ -16868,11 +16848,8 @@ func (e *Engine) writeBatchWithOutcomeContext( } werr = e.db.ReplaceSessionMessages(s.ID, msgs) } else { - if !e.disableSignalRecompute { - update, findings = e.computeSignalsAndSecretsForStorage(s, msgs) - if ctx.Err() != nil { - return outcome - } + if !e.disableSignalRecompute && ctx.Err() != nil { + return outcome } werr = e.writeMessages(s.ID, msgs) } @@ -16892,14 +16869,8 @@ func (e *Engine) writeBatchWithOutcomeContext( if ctx.Err() != nil { return outcome } - usageEvents, usageErr := e.usageEventsForWriteContext( - ctx, s.ID, pw.usageEvents, - ) - if usageErr != nil { - return outcome - } if err := e.db.ReplaceSessionUsageEvents( - s.ID, usageEvents, + s.ID, prepared.UsageEvents, ); err != nil { if ctx.Err() != nil { return outcome @@ -16989,10 +16960,10 @@ func (e *Engine) prepareSessionWrite( pw pendingWrite, resolveWorktreeProject worktreeProjectResolver, ) (db.Session, []db.Message, sessionWriteVerdict) { - s, msgs, verdict, _ := e.prepareSessionWriteContext( - context.Background(), pw, resolveWorktreeProject, + prepared, verdict, _ := e.prepareSessionNormalizedContext( + context.Background(), pw, resolveWorktreeProject, false, ) - return s, msgs, verdict + return prepared.Session, prepared.Messages, verdict } func (e *Engine) prepareSessionWriteContext( @@ -17000,22 +16971,31 @@ func (e *Engine) prepareSessionWriteContext( pw pendingWrite, resolveWorktreeProject worktreeProjectResolver, ) (db.Session, []db.Message, sessionWriteVerdict, error) { - msgs, err := toDBMessagesContext(ctx, pw, e.blockedResultCategories) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - msgs, _ = e.db.ProjectToolResultImages(msgs) - s, err := toDBSessionContext(ctx, pw) + prepared, verdict, err := e.prepareSessionNormalizedContext( + ctx, pw, resolveWorktreeProject, false, + ) + return prepared.Session, prepared.Messages, verdict, err +} + +func (e *Engine) prepareSessionNormalizedContext( + ctx context.Context, + pw pendingWrite, + resolveWorktreeProject worktreeProjectResolver, + derive bool, +) (ingest.PreparedSession, sessionWriteVerdict, error) { + candidate, err := ingest.PrepareCandidate(ctx, parser.ParseResult{ + Session: pw.sess, Messages: pw.msgs, UsageEvents: pw.usageEvents, + }, ingest.ContentOptions{ + BlockedResultCategories: e.blockedResultCategories, + ToolResultImages: e.toolResultImages, + ArchiveContent: e.db.ArchiveContent(), + }) if err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - if err := applySessionMessageDerivedFieldsContext( - ctx, &s, msgs, pw.sess.CountsAuthoritative, - ); err != nil { - return db.Session{}, nil, sessionWritePreserved, err + return ingest.PreparedSession{}, sessionWritePreserved, err } + s, msgs := candidate.Session, candidate.Messages if err := e.applyRemoteRewritesContext(ctx, &s, msgs); err != nil { - return db.Session{}, nil, sessionWritePreserved, err + return ingest.PreparedSession{}, sessionWritePreserved, err } applySourceCwdResolution( &s, pw.sourceCwdResolution, pw.sourceCwdStored, pw.sourceCwdStoredOK, @@ -17033,81 +17013,40 @@ func (e *Engine) prepareSessionWriteContext( // preserve/merge handling so a filtered session is not written by // any downstream path. if !e.cwdFilter.allows(s.Cwd) { - return db.Session{}, nil, sessionWriteCwdFiltered, nil + return ingest.PreparedSession{}, sessionWriteCwdFiltered, nil } if err := ctx.Err(); err != nil { - return db.Session{}, nil, sessionWritePreserved, err + return ingest.PreparedSession{}, sessionWritePreserved, err } - if e.shouldPreserveOpenCodeFormatArchive( - pw.sess.Agent, pw.sess.File.Path, s.ID, - pw.sess.File.Mtime, derefString(s.FileHash), msgs, - ) { - return db.Session{}, nil, sessionWritePreserved, nil - } - if e.shouldPreserveRooCodeArchive(pw.sess.Agent, s.ID, msgs) { - return db.Session{}, nil, sessionWritePreserved, nil - } - if mergedMsgs, preserve, archived := e.reconcileVisualStudioCopilotArchive( - pw.sess.Agent, s.ID, pw.sess.File.Size, msgs, - ); preserve { - return db.Session{}, nil, sessionWritePreserved, nil - } else if mergedMsgs != nil { - parsedMsgs := msgs - msgs = mergedMsgs - msgs, _ = e.db.ProjectToolResultImages(msgs) - applyVisualStudioCopilotArchiveSessionFields( - &s, archived, parsedMsgs, msgs, - ) - if err := applySessionMessageDerivedFieldsContext( - ctx, &s, msgs, pw.sess.CountsAuthoritative, - ); err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - if err := applySessionTokenTotalsFromMessagesContext( - ctx, &s, msgs, - ); err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } + candidate.Session, candidate.Messages = s, msgs + history, err := e.reconcileProviderHistoryContext(ctx, candidate) + if err != nil { + return ingest.PreparedSession{}, sessionWritePreserved, err } + if history.Action == ingest.HistoryPreserve { + return ingest.PreparedSession{}, sessionWritePreserved, nil + } + candidate = history.Candidate if err := ctx.Err(); err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - // Snapshot, before sanitizing, whether the session's token aggregates - // are derived from the per-message rows or the per-usage-event rows, by - // matching the stored value against each source's raw sum/max. Aggregates - // set directly from a session-level usage summary -- agents like - // Warp/Vibe/Hermes/Zed -- must survive the per-row clamp untouched. - // Source=="session" usage events mirror those same summary totals, so - // exclude them from the event-derived detector and re-clamp path. - msgTotal, msgHasOut, msgPeak, msgHasCtx, err := - messageTokenTotalsContext(ctx, msgs) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err + return ingest.PreparedSession{}, sessionWritePreserved, err + } + options := ingest.ContentOptions{ + BlockedResultCategories: e.blockedResultCategories, + ToolResultImages: e.toolResultImages, + ArchiveContent: e.db.ArchiveContent(), + } + var prepared ingest.PreparedSession + if !derive || pw.staged != nil || e.disableSignalRecompute { + prepared, err = ingest.FinalizeRows(ctx, candidate, options) + } else { + prepared, err = ingest.Finalize(ctx, candidate, options) } - evtTotal, evtHasOut, evtPeak, evtHasCtx, err := - usageEventTokenTotalsContext(ctx, pw.usageEvents, false) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - totalFromMsgs := s.HasTotalOutputTokens == msgHasOut && - s.TotalOutputTokens == msgTotal - totalFromEvts := s.HasTotalOutputTokens == evtHasOut && - s.TotalOutputTokens == evtTotal - peakFromMsgs := s.HasPeakContextTokens == msgHasCtx && - s.PeakContextTokens == msgPeak - peakFromEvts := s.HasPeakContextTokens == evtHasCtx && - s.PeakContextTokens == evtPeak - - // Central validation/sanitization pass: every session write flows - // through here so all agents are covered uniformly. The returned fix - // counts and the parser malformed-line count are accumulated per - // agent for the sync summary's anomaly section. - vs, err := validateAndSanitizeContext(ctx, &s, msgs, nil) if err != nil { - return db.Session{}, nil, sessionWritePreserved, err + return ingest.PreparedSession{}, sessionWritePreserved, err } - e.anomalies.recordSanitize(vs) + s = prepared.Session + e.anomalies.recordSanitize(prepared.Validation) e.anomalies.recordMalformedLines( s.Agent, pw.sess.File.Path, s.ParserMalformedLines, ) @@ -17126,324 +17065,97 @@ func (e *Engine) prepareSessionWriteContext( if pw.sess.GenMetadataWithoutUsage { e.anomalies.recordGenMetadataWithoutUsageSession(s.Agent) } + return prepared, sessionWriteOK, ctx.Err() +} - // A per-row token clamp must not leave an inflated value stranded in a - // row-derived session total while the row that produced it was clamped. - // Re-derive a matched aggregate from its now-clamped source (messages - // clamped above; usage events clamped on the fly the same way - // toDBUsageEvents will store them). Summary-derived aggregates match - // neither source and are left as-is. The sum is re-summed from clamped - // rows rather than clamped to the per-row bound, so a legitimately large - // total over many rows is preserved. Re-deriving is a no-op when nothing - // was clamped, keeping the pass idempotent. Messages take precedence when - // both sources match (identical values). - if totalFromMsgs { - t, h, _, _, err := messageTokenTotalsContext(ctx, msgs) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - s.TotalOutputTokens, s.HasTotalOutputTokens = t, h - } else if totalFromEvts { - t, h, _, _, err := usageEventTokenTotalsContext( - ctx, pw.usageEvents, true, +func (e *Engine) reconcileProviderHistoryContext( + ctx context.Context, candidate ingest.Candidate, +) (ingest.HistoryResult, error) { + agent := candidate.Parsed.Session.Agent + options := ingest.ContentOptions{ + BlockedResultCategories: e.blockedResultCategories, + ToolResultImages: e.toolResultImages, + ArchiveContent: e.db.ArchiveContent(), + } + if ingest.IsClaudeFormatTranscript( + agent, candidate.Parsed.Session.File.Path, + ) { + return ingest.ReconcileProviderHistory( + ctx, candidate, nil, options, ) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err - } - s.TotalOutputTokens, s.HasTotalOutputTokens = t, h } - if peakFromMsgs { - _, _, p, h, err := messageTokenTotalsContext(ctx, msgs) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err + var prior *ingest.PriorSession + switch agent { + case parser.AgentRooCode, parser.AgentKiloLegacy: + if len(candidate.Messages) > 0 { + break } - s.PeakContextTokens, s.HasPeakContextTokens = p, h - } else if peakFromEvts { - _, _, p, h, err := usageEventTokenTotalsContext( - ctx, pw.usageEvents, true, - ) - if err != nil { - return db.Session{}, nil, sessionWritePreserved, err + store := e.archiveStore + if store == nil { + store = e.db } - s.PeakContextTokens, s.HasPeakContextTokens = p, h - } - return s, msgs, sessionWriteOK, ctx.Err() -} - -func applySessionMessageDerivedFieldsContext( - ctx context.Context, - s *db.Session, - msgs []db.Message, - countsAuthoritative bool, -) error { - if !countsAuthoritative { - var err error - s.MessageCount, s.UserMessageCount, err = postFilterCountsContext( - ctx, msgs, + messages, err := store.GetAllMessages( + context.Background(), candidate.Session.ID, ) - if err != nil { - return err + if err == nil && len(messages) > 0 { + prior = &ingest.PriorSession{ + Session: candidate.Session, Messages: messages, + } } - } - if err := ctx.Err(); err != nil { - return err - } - s.IsAutomated = db.IsAutomatedSessionMetadata(s.Agent, s.SessionKind) || - db.IsAutomatedTranscript( - s.UserMessageCount, msgs, s.FirstMessage, + case parser.AgentVSCopilot: + stored, err := e.db.GetSessionFull( + context.Background(), candidate.Session.ID, ) - return ctx.Err() -} - -// messageTokenTotalsContext computes the message-derived session token -// aggregates: the sum of per-message output tokens and the peak -// per-message context tokens, each with a presence flag. It is the -// canonical derivation shared by session preparation and the post-sanitize -// reconciliation that re-derives message-derived totals from the clamped rows. -// Absent values return 0 with a false presence. -func messageTokenTotalsContext( - ctx context.Context, msgs []db.Message, -) (totalOut int, hasOut bool, peakCtx int, hasCtx bool, err error) { - for _, msg := range msgs { - if err = ctx.Err(); err != nil { - return - } - if msg.HasOutputTokens { - hasOut = true - totalOut += msg.OutputTokens - } - if msg.HasContextTokens { - hasCtx = true - if msg.ContextTokens > peakCtx { - peakCtx = msg.ContextTokens + if err == nil && stored != nil { + messages, messagesErr := e.db.GetAllMessages( + context.Background(), candidate.Session.ID, + ) + if messagesErr == nil && len(messages) > 0 { + prior = &ingest.PriorSession{ + Session: *stored, Messages: messages, + } } } - } - err = ctx.Err() - return -} - -func applySessionTokenTotalsFromMessagesContext( - ctx context.Context, s *db.Session, msgs []db.Message, -) error { - totalOut, hasOut, peakCtx, hasCtx, err := messageTokenTotalsContext( - ctx, msgs, - ) - if err != nil { - return err - } - s.TotalOutputTokens = totalOut - s.HasTotalOutputTokens = hasOut - s.PeakContextTokens = peakCtx - s.HasPeakContextTokens = hasCtx - return nil -} - -// usageEventTokenTotals computes event-derived session token aggregates through -// parser.UsageEventTokenAggregate -- the same rollup per-turn event parsers use -// to populate stored session totals (positive output summed, peak full context -// = input + cache-creation + cache-read where positive). Session-summary usage -// events mirror parser summary totals rather than per-turn rows, so they are -// excluded from this detector and re-clamp path. When clamp is true each -// included event token field is first bounded to the per-row plausibility cap, -// matching how sanitizeUsageEvent bounds the stored usage_event row. -func usageEventTokenTotalsContext( - ctx context.Context, events []parser.ParsedUsageEvent, clamp bool, -) (totalOut int, hasOut bool, peakCtx int, hasCtx bool, err error) { - rolled := make([]parser.ParsedUsageEvent, 0, len(events)) - for _, ev := range events { - if err = ctx.Err(); err != nil { - return - } - if ev.Source == "session" { - continue - } - rolled = append(rolled, ev) - } - if clamp { - for i, ev := range rolled { - if err = ctx.Err(); err != nil { - return + default: + if ingest.IsOpenCodeFormatStorageAgent(agent) { + store := e.archiveStore + if store == nil { + store = e.db } - ev.InputTokens = clampedTokens(ev.InputTokens) - ev.OutputTokens = clampedTokens(ev.OutputTokens) - ev.CacheCreationInputTokens = clampedTokens( - ev.CacheCreationInputTokens, + stored, err := store.GetSessionFull( + context.Background(), candidate.Session.ID, ) - ev.CacheReadInputTokens = clampedTokens(ev.CacheReadInputTokens) - rolled[i] = ev - } - } - totalOut, hasOut, peakCtx, hasCtx, err = - parser.UsageEventTokenAggregateContext(ctx, rolled) - return -} - -func applyVisualStudioCopilotArchiveSessionFields( - s *db.Session, archived *db.Session, - parsedMsgs, mergedMsgs []db.Message, -) { - if archived == nil { - return - } - archiveExtendsBounds := sessionTimeBefore( - archived.StartedAt, s.StartedAt, - ) || sessionTimeAfter(archived.EndedAt, s.EndedAt) - if !visualStudioCopilotMergedFirstMessageFromParsed( - parsedMsgs, mergedMsgs, - ) { - s.FirstMessage = cloneStringPtr(archived.FirstMessage) - } - if archiveExtendsBounds || stringPtrEmpty(s.SessionName) { - s.SessionName = cloneStringPtr(archived.SessionName) - } - s.StartedAt = earlierSessionTime(archived.StartedAt, s.StartedAt) - s.EndedAt = laterSessionTime(archived.EndedAt, s.EndedAt) -} - -func visualStudioCopilotMergedFirstMessageFromParsed( - parsed, merged []db.Message, -) bool { - if len(parsed) == 0 || len(merged) == 0 { - return false - } - mergedFirst := merged[0] - for _, parsedMsg := range parsed { - if visualStudioCopilotMessagePresenceKey(parsedMsg) != - visualStudioCopilotMessagePresenceKey(mergedFirst) { - continue + if err == nil && stored != nil { + messages, messagesErr := store.GetAllMessages( + context.Background(), candidate.Session.ID, + ) + if messagesErr == nil && len(messages) > 0 { + prior = &ingest.PriorSession{ + Session: *stored, Messages: messages, + } + } + } } - return !visualStudioCopilotMessageLooksIncomplete( - parsedMsg, mergedFirst, - ) && !visualStudioCopilotMessageHasArchiveUpdate( - mergedFirst, parsedMsg, - ) - } - return false -} - -func stringPtrEmpty(v *string) bool { - return v == nil || strings.TrimSpace(*v) == "" -} - -func cloneStringPtr(v *string) *string { - if v == nil { - return nil - } - clone := *v - return &clone -} - -func sessionTimeBefore(a, b *string) bool { - return sessionTimeCompares(a, b, func(aTime, bTime time.Time) bool { - return aTime.Before(bTime) - }) -} - -func sessionTimeAfter(a, b *string) bool { - return sessionTimeCompares(a, b, func(aTime, bTime time.Time) bool { - return aTime.After(bTime) - }) -} - -func sessionTimeCompares( - a, b *string, compare func(time.Time, time.Time) bool, -) bool { - if a == nil || b == nil { - return a != nil && b == nil - } - aTime, aErr := time.Parse(time.RFC3339Nano, *a) - bTime, bErr := time.Parse(time.RFC3339Nano, *b) - if aErr != nil || bErr != nil { - return false - } - return compare(aTime, bTime) -} - -func earlierSessionTime(a, b *string) *string { - return chooseSessionTime(a, b, func(aTime, bTime time.Time) bool { - return aTime.Before(bTime) - }) -} - -func laterSessionTime(a, b *string) *string { - return chooseSessionTime(a, b, func(aTime, bTime time.Time) bool { - return aTime.After(bTime) - }) -} - -func chooseSessionTime( - a, b *string, chooseA func(time.Time, time.Time) bool, -) *string { - switch { - case a == nil: - return cloneStringPtr(b) - case b == nil: - return cloneStringPtr(a) } - aTime, aErr := time.Parse(time.RFC3339Nano, *a) - bTime, bErr := time.Parse(time.RFC3339Nano, *b) - switch { - case aErr != nil: - return cloneStringPtr(b) - case bErr != nil: - return cloneStringPtr(a) - case chooseA(aTime, bTime): - return cloneStringPtr(a) - default: - return cloneStringPtr(b) - } -} - -// reconcileVisualStudioCopilotArchive returns either a preserved-archive skip -// or a merged transcript for an incomplete Visual Studio Copilot reparse. A -// conversation's transcript is rebuilt from every sibling trace file and -// written with full message replacement, so when a sibling is rotated away or -// deleted the reparse can see fewer spans or weaker span metadata and would -// otherwise drop messages and tool results already stored in SQLite. If a -// remaining trace gained richer data or new messages, merge those updates into -// the archived transcript while retaining archived-only messages. -func (e *Engine) reconcileVisualStudioCopilotArchive( - agent parser.AgentType, sessionID string, - currentSize int64, currentMsgs []db.Message, -) (merged []db.Message, preserve bool, archived *db.Session) { - if agent != parser.AgentVSCopilot { - return nil, false, nil - } - stored, err := e.db.GetSessionFull(context.Background(), sessionID) - if err != nil || stored == nil { - return nil, false, nil - } - storedSize := derefInt64(stored.FileSize) - storedMsgs, err := e.db.GetAllMessages(context.Background(), sessionID) - if err != nil || len(storedMsgs) == 0 { - return nil, false, nil - } - decision := visualStudioCopilotArchiveDecision( - currentMsgs, storedMsgs, + result, err := ingest.ReconcileProviderHistory( + ctx, candidate, prior, options, ) - if decision.preserve { + if err != nil || prior == nil { + return result, err + } + switch result.Action { + case ingest.HistoryPreserve: log.Printf( - "preserve %s %s: reparse looks incomplete relative to archived "+ - "transcript (%d stored messages, %d parsed messages, "+ - "composite trace %d->%d bytes)", - agent, sessionID, len(storedMsgs), len(currentMsgs), - storedSize, currentSize, + "preserve %s %s: reparse looks incomplete relative to archived transcript (%d stored messages, %d parsed messages)", + agent, candidate.Session.ID, len(prior.Messages), len(candidate.Messages), ) - return storedMsgs, false, stored - } - if decision.merged != nil { + case ingest.HistoryMerge: log.Printf( - "merge %s %s: reparse updated archived messages while "+ - "retaining archived transcript rows (%d stored "+ - "messages, %d parsed messages, composite trace "+ - "%d->%d bytes)", - agent, sessionID, len(storedMsgs), len(currentMsgs), - storedSize, currentSize, + "merge %s %s: reparse updated archived messages while retaining archived transcript rows (%d stored messages, %d parsed messages)", + agent, candidate.Session.ID, len(prior.Messages), len(candidate.Messages), ) - return decision.merged, false, stored } - return nil, false, nil + return result, nil } type visualStudioCopilotArchiveReconcile struct { @@ -17454,434 +17166,44 @@ type visualStudioCopilotArchiveReconcile struct { func visualStudioCopilotArchiveDecision( parsed, stored []db.Message, ) visualStudioCopilotArchiveReconcile { - if len(stored) == 0 { - return visualStudioCopilotArchiveReconcile{} - } - if parsed == nil { - return visualStudioCopilotArchiveReconcile{preserve: true} - } - - storedByKey := make(map[string][]int, len(stored)) - for i, msg := range stored { - key := visualStudioCopilotMessagePresenceKey(msg) - storedByKey[key] = append(storedByKey[key], i) - } - matchedStored := make([]bool, len(stored)) - updates := make(map[int]db.Message) - additions := make([]db.Message, 0) - hasIncomplete := false - for _, parsedMsg := range parsed { - key := visualStudioCopilotMessagePresenceKey(parsedMsg) - candidates := storedByKey[key] - if len(candidates) == 0 { - additions = append(additions, parsedMsg) - continue - } - storedIndex := candidates[0] - storedByKey[key] = candidates[1:] - matchedStored[storedIndex] = true - storedMsg := stored[storedIndex] - incomplete := visualStudioCopilotMessageLooksIncomplete( - parsedMsg, storedMsg, - ) - if incomplete { - hasIncomplete = true - } - if !incomplete && - visualStudioCopilotMessageHasArchiveUpdate( - parsedMsg, storedMsg, - ) { - updates[storedIndex] = parsedMsg - } - } - fallbackMatched := false - additions, fallbackMatched = visualStudioCopilotResolveArchiveAdditions( - stored, matchedStored, updates, additions, &hasIncomplete, + merged, preserve := ingest.ReconcileVisualStudioCopilotArchive( + parsed, stored, ) - hasArchiveOnly := false - for _, matched := range matchedStored { - if !matched { - hasArchiveOnly = true - break - } - } - if hasIncomplete || hasArchiveOnly || fallbackMatched { - if len(updates) > 0 || len(additions) > 0 || - (fallbackMatched && !hasIncomplete) { - return visualStudioCopilotArchiveReconcile{ - merged: visualStudioCopilotMergeArchiveMessages( - stored, updates, additions, - ), - } - } - return visualStudioCopilotArchiveReconcile{preserve: true} - } - return visualStudioCopilotArchiveReconcile{} -} - -func visualStudioCopilotResolveArchiveAdditions( - stored []db.Message, - matchedStored []bool, - updates map[int]db.Message, - additions []db.Message, - hasIncomplete *bool, -) ([]db.Message, bool) { - matched := false - unresolved := additions[:0] - for _, parsedMsg := range additions { - storedIndex, ok := visualStudioCopilotArchiveFallbackMatch( - parsedMsg, stored, matchedStored, - ) - if !ok { - unresolved = append(unresolved, parsedMsg) - continue - } - matched = true - matchedStored[storedIndex] = true - storedMsg := stored[storedIndex] - incomplete := visualStudioCopilotMessageLooksIncomplete( - parsedMsg, storedMsg, - ) - if incomplete { - *hasIncomplete = true - continue - } - update := visualStudioCopilotArchiveFallbackUpdate( - parsedMsg, storedMsg, - ) - if visualStudioCopilotMessageHasArchiveUpdate(update, storedMsg) { - updates[storedIndex] = update - } - } - return unresolved, matched -} - -func visualStudioCopilotArchiveFallbackMatch( - parsed db.Message, - stored []db.Message, - matchedStored []bool, -) (int, bool) { - match := -1 - for i, storedMsg := range stored { - if matchedStored[i] { - continue - } - if !visualStudioCopilotMessagesFallbackMatch(parsed, storedMsg) { - continue - } - if match != -1 { - return 0, false - } - match = i - } - if match == -1 { - return 0, false + return visualStudioCopilotArchiveReconcile{ + preserve: preserve, merged: merged, } - return match, true } -func visualStudioCopilotMessagesFallbackMatch( +func visualStudioCopilotMessageLooksIncomplete( parsed, stored db.Message, ) bool { - if parsed.Role != stored.Role { - return false - } - if visualStudioCopilotMessagesShareToolIdentity(parsed, stored) { - return true - } - return visualStudioCopilotMessagesShareContentIdentity(parsed, stored) + return ingest.VisualStudioCopilotMessageLooksIncomplete(parsed, stored) } -func visualStudioCopilotMessagesShareToolIdentity( +func visualStudioCopilotMessageHasArchiveUpdate( parsed, stored db.Message, ) bool { - if len(parsed.ToolCalls) == 0 || len(stored.ToolCalls) == 0 { - return false - } - parsedIDs := make(map[string]string, len(parsed.ToolCalls)) - for _, call := range parsed.ToolCalls { - id := strings.TrimSpace(call.ToolUseID) - if id == "" { - continue - } - parsedIDs[id] = strings.TrimSpace(call.ToolName) - } - for _, call := range stored.ToolCalls { - id := strings.TrimSpace(call.ToolUseID) - if id == "" { - continue - } - parsedName, ok := parsedIDs[id] - if !ok { - continue - } - storedName := strings.TrimSpace(call.ToolName) - if parsedName != "" && storedName != "" && - parsedName != storedName { - continue - } - return true - } - return false + return ingest.VisualStudioCopilotMessageHasArchiveUpdate(parsed, stored) } -func visualStudioCopilotMessagesShareContentIdentity( - parsed, stored db.Message, -) bool { - if len(parsed.ToolCalls) > 0 || len(stored.ToolCalls) > 0 { - return false - } - switch parsed.Role { - case string(parser.RoleAssistant), string(parser.RoleUser): - default: - return false - } - return parsed.Content != "" && parsed.Content == stored.Content +type batchSourceFile struct { + path string + mtime int64 + fingerprint string } -func visualStudioCopilotArchiveFallbackUpdate( - parsed, stored db.Message, -) db.Message { - update := parsed - // A duplicate span can be flushed later with a different timestamp; keep - // the archived timestamp as the transcript anchor while taking any richer - // parsed payload such as tool results or token usage. - update.Timestamp = stored.Timestamp - return update -} - -func visualStudioCopilotMergeArchiveMessages( - stored []db.Message, updates map[int]db.Message, - additions []db.Message, -) []db.Message { - merged := make([]db.Message, 0, len(stored)+len(additions)) - merged = append(merged, stored...) - for index, msg := range updates { - merged[index] = msg - } - merged = append(merged, additions...) - if len(additions) > 0 { - slices.SortStableFunc( - merged, compareVisualStudioCopilotMessageOrder, - ) - } - for i := range merged { - merged[i].Ordinal = i - } - return merged -} - -func compareVisualStudioCopilotMessageOrder(a, b db.Message) int { - aTime, aOK := visualStudioCopilotMessageTime(a) - bTime, bOK := visualStudioCopilotMessageTime(b) - if aOK && bOK { - switch { - case aTime.Before(bTime): - return -1 - case aTime.After(bTime): - return 1 - default: - return 0 - } - } - if aOK { - return -1 - } - if bOK { - return 1 - } - return 0 -} - -func visualStudioCopilotMessageTime(msg db.Message) (time.Time, bool) { - if msg.Timestamp == "" { - return time.Time{}, false - } - parsed, err := time.Parse(time.RFC3339Nano, msg.Timestamp) - if err != nil { - return time.Time{}, false - } - return parsed, true -} - -func visualStudioCopilotMessagePresenceKey(msg db.Message) string { - if msg.Timestamp != "" { - return msg.Role + "\x00time\x00" + msg.Timestamp - } - if msg.SourceUUID != "" { - return msg.Role + "\x00source\x00" + msg.SourceUUID - } - return fmt.Sprintf("%s\x00ordinal\x00%d", msg.Role, msg.Ordinal) -} - -func visualStudioCopilotMessageLooksIncomplete( - parsed, stored db.Message, -) bool { - if parsed.Role != stored.Role { - return false - } - // Stored rows are sanitized and length-adjusted on write; measure the - // parsed side the same way so a reparse that only stripped control bytes - // is not judged shorter and allowed to bypass archive preservation. - p := sanitizedForArchiveCompare(parsed) - if p.ContentLength < stored.ContentLength { - return true - } - if stored.HasThinking && !p.HasThinking { - return true - } - if stored.HasOutputTokens && - (!p.HasOutputTokens || - p.OutputTokens < stored.OutputTokens) { - return true - } - if stored.HasContextTokens && - (!p.HasContextTokens || - p.ContextTokens < stored.ContextTokens) { - return true - } - if len(p.ToolCalls) < len(stored.ToolCalls) { - return true - } - if countToolResultEvents(p.ToolCalls) < - countToolResultEvents(stored.ToolCalls) { - return true - } - return countToolResultContentLength(p.ToolCalls) < - countToolResultContentLength(stored.ToolCalls) -} - -// sanitizedForArchiveCompare returns a copy of m with the same -// validation/sanitization stored rows receive on write (control runes -// stripped, ContentLength delta-adjusted, tokens clamped), so the VS Copilot -// archive reconcile compares freshly parsed messages against archived rows -// like-for-like. The copy is shallow; sanitizeMessage only rewrites value -// fields, leaving the shared ToolCalls slice untouched. -func sanitizedForArchiveCompare(m db.Message) db.Message { - _ = sanitizeMessage(&m) - return m -} - -func visualStudioCopilotMessageHasArchiveUpdate( - parsed, stored db.Message, -) bool { - if parsed.Role != stored.Role { - return false - } - // Stored rows are sanitized and length-adjusted on write, but the parsed - // message still carries raw content here. Compare a sanitized copy so a - // reparse that differs only in stripped control bytes is not treated as - // an archive update, preserving idempotency. - p := sanitizedForArchiveCompare(parsed) - if p.ContentLength > stored.ContentLength { - return true - } - if p.ContentLength == stored.ContentLength && - p.Content != stored.Content { - return true - } - if p.HasThinking && (!stored.HasThinking || - p.ThinkingText != stored.ThinkingText) { - return true - } - if p.HasOutputTokens && - (!stored.HasOutputTokens || - p.OutputTokens > stored.OutputTokens) { - return true - } - if p.HasContextTokens && - (!stored.HasContextTokens || - p.ContextTokens > stored.ContextTokens) { - return true - } - if string(p.TokenUsage) != "" && - string(p.TokenUsage) != string(stored.TokenUsage) { - return true - } - return visualStudioCopilotToolCallsHaveArchiveUpdate( - p.ToolCalls, stored.ToolCalls, - ) -} - -func visualStudioCopilotToolCallsHaveArchiveUpdate( - parsed, stored []db.ToolCall, -) bool { - if len(parsed) > len(stored) { - return true - } - for i := 0; i < len(parsed) && i < len(stored); i++ { - if visualStudioCopilotToolCallHasArchiveUpdate( - parsed[i], stored[i], - ) { - return true - } - } - return false -} - -func visualStudioCopilotToolCallHasArchiveUpdate( - parsed, stored db.ToolCall, -) bool { - if parsed.ResultContentLength > stored.ResultContentLength { - return true - } - if parsed.ResultContentLength == stored.ResultContentLength && - parsed.ResultContent != "" && - parsed.ResultContent != stored.ResultContent { - return true - } - if len(parsed.ResultEvents) > len(stored.ResultEvents) { - return true - } - for i := 0; i < len(parsed.ResultEvents) && - i < len(stored.ResultEvents); i++ { - parsedEvent := parsed.ResultEvents[i] - storedEvent := stored.ResultEvents[i] - if parsedEvent.ContentLength > storedEvent.ContentLength { - return true - } - if parsedEvent.ContentLength == storedEvent.ContentLength && - parsedEvent.Content != "" && - parsedEvent.Content != storedEvent.Content { - return true - } - if parsedEvent.Status != "" && - parsedEvent.Status != storedEvent.Status { - return true - } - } - return false -} - -func countToolResultContentLength(calls []db.ToolCall) int { - total := 0 - for _, call := range calls { - total += call.ResultContentLength - for _, event := range call.ResultEvents { - total += event.ContentLength - } - } - return total -} - -type batchSourceFile struct { - path string - mtime int64 - fingerprint string -} - -type projectIdentityCacheEntry struct { - rootPath string - repositoryPath string - gitDir string - gitRemoteName string - gitRemote string - remoteResolution export.ProjectResolution - remoteCandidates int - worktreeName string - worktreeRootPath string - worktreeKind export.WorktreeRelationship - expiresAt time.Time +type projectIdentityCacheEntry struct { + rootPath string + repositoryPath string + gitDir string + gitRemoteName string + gitRemote string + remoteResolution export.ProjectResolution + remoteCandidates int + worktreeName string + worktreeRootPath string + worktreeKind export.WorktreeRelationship + expiresAt time.Time } type localGitIdentity struct { @@ -17993,8 +17315,8 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( return outcome } tPrep := time.Now() - s, msgs, verdict, prepErr := e.prepareSessionWriteContext( - ctx, pw, resolveWorktreeProject, + prepared, verdict, prepErr := e.prepareSessionNormalizedContext( + ctx, pw, resolveWorktreeProject, true, ) if prepErr != nil { return outcome @@ -18006,6 +17328,7 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( } continue } + s, msgs := prepared.Session, prepared.Messages if pw.staged != nil { // Staged streaming results bypass the bulk batch: their // tool-result rows live in the staging scratch database and @@ -18045,7 +17368,7 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( continue } if err := e.db.ReplaceSessionUsageEvents( - s.ID, e.usageEventsForWrite(s.ID, pw.usageEvents), + s.ID, prepared.UsageEvents, ); err != nil { log.Printf( "write usage events for %s: %v", s.ID, err, @@ -18079,12 +17402,12 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( replaceMessages := shouldReplaceFullParseMessages( pw, forceReplace, false, false, ) - var update db.SessionSignalUpdate - var findings []db.SecretFinding + update := prepared.Signals + findings := prepared.Findings if !e.disableSignalRecompute { tScan := time.Now() var signalErr error - update, findings, signalErr = e.computeFullSignalsAndSecretsForStorage(s, msgs, nil) + update, signalErr = e.attachFullSignalState(s, msgs, update, nil) if signalErr != nil { log.Printf("compute full signals %s: %v", s.ID, signalErr) outcome.failedSessions++ @@ -18110,18 +17433,12 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( checkpoint, checkpointBlobs = nil, nil } } - usageEvents, usageErr := e.usageEventsForWriteContext( - ctx, s.ID, pw.usageEvents, - ) - if usageErr != nil { - return outcome - } identityObservation, hasIdentityObservation := e.projectIdentityObservationForWrite(pw, s) writes = append(writes, db.SessionBatchWrite{ Session: s, Messages: msgs, - UsageEvents: usageEvents, + UsageEvents: prepared.UsageEvents, IdentityObservation: identityObservationOrZero( identityObservation, hasIdentityObservation, ), @@ -19019,12 +18336,16 @@ func (e *Engine) writeSessionFullWithResolver( return err } pw = preserved[0] - s, msgs, verdict := e.prepareSessionWrite( - pw, resolveWorktreeProject, + prepared, verdict, err := e.prepareSessionNormalizedContext( + ctx, pw, resolveWorktreeProject, true, ) + if err != nil { + return err + } if verdict != sessionWriteOK { return errSessionPreserved } + s, msgs := prepared.Session, prepared.Messages _, err = e.upsertSessionPendingContentForWrite(pw, s) if err != nil { if isIntentionalSessionSkip(err) { @@ -19062,7 +18383,9 @@ func (e *Engine) writeSessionFullWithResolver( return err } } else { - update, findings, signalErr := e.computeFullSignalsAndSecretsForStorage(s, msgs, nil) + update, signalErr := e.attachFullSignalState( + s, msgs, prepared.Signals, nil, + ) if signalErr != nil { return signalErr } @@ -19081,7 +18404,7 @@ func (e *Engine) writeSessionFullWithResolver( } } if err := e.db.ReplaceSessionContentWithCheckpoint( - s.ID, msgs, update, findings, checkpoint, checkpointBlobs, + s.ID, msgs, update, prepared.Findings, checkpoint, checkpointBlobs, ); err != nil { log.Printf( "replace messages for %s: %v", @@ -19091,7 +18414,7 @@ func (e *Engine) writeSessionFullWithResolver( } } if err := e.db.ReplaceSessionUsageEvents( - s.ID, e.usageEventsForWrite(s.ID, pw.usageEvents), + s.ID, prepared.UsageEvents, ); err != nil { log.Printf( "replace usage events for %s: %v", @@ -19117,170 +18440,12 @@ func (e *Engine) writeSessionFullWithResolver( return nil } -// shouldPreserveRooCodeArchive reports whether a zero-message RooCode -// parse must not overwrite an archived transcript. A vanished (or -// torn) ui_messages.json parses as a zero-message session while -// history_item.json keeps the task discoverable, so writing that -// parse would corrupt the session's counts on normal sync and — with -// RooCode on the full-replace path — delete the archived messages -// outright; on a rebuild it would recreate the session empty in the -// fresh DB, which also blocks the orphan-copy pass from restoring it. -// Newly created metadata-only tasks have no archived messages and -// still write normally. -func (e *Engine) shouldPreserveRooCodeArchive( - agent parser.AgentType, sessionID string, msgs []db.Message, -) bool { - if (agent != parser.AgentRooCode && agent != parser.AgentKiloLegacy) || len(msgs) > 0 { - return false - } - store := e.archiveStore - if store == nil { - store = e.db - } - stored, err := store.GetAllMessages(context.Background(), sessionID) - if err != nil || len(stored) == 0 { - return false - } - log.Printf( - "skip %s session %s: transcript parsed empty but archive has %d messages", - agent, sessionID, len(stored), - ) - return true -} - -func (e *Engine) shouldPreserveOpenCodeFormatArchive( - agent parser.AgentType, path, sessionID string, - currentMtime int64, - currentHash string, - currentMsgs []db.Message, -) bool { - if !isOpenCodeFormatStorageAgent(agent) { - return false - } - // An ICodeMate CLI transcript is a plain rewritable file, never a - // self-preserving OpenCode container. - if isClaudeFormatTranscript(agent, path) { - return false - } - store := e.archiveStore - if store == nil { - store = e.db - } - stored, err := store.GetSessionFull( - context.Background(), sessionID, - ) - if err != nil || stored == nil { - return false - } - storedHash := derefString(stored.FileHash) - storedPath := derefString(stored.FilePath) - storedMtime := derefInt64(stored.FileMtime) - storedHasStorageFingerprint := hasOpenCodeFormatStorageFingerprint( - agent, storedHash, - ) - currentIsOpenCodeStorage := isOpenCodeFormatStoragePath(agent, path) || - isOpenCodeFormatSQLiteVirtualPath(agent, path) - storedIsOpenCodeStorage := isOpenCodeFormatStoragePath(agent, storedPath) || - isOpenCodeFormatSQLiteVirtualPath(agent, storedPath) || - (storedPath == "" && storedHasStorageFingerprint) - if !currentIsOpenCodeStorage && !storedIsOpenCodeStorage { - return false - } - storedIsSQLiteVirtual := isOpenCodeFormatSQLiteVirtualPath( - agent, storedPath, - ) - storedIsStorageArchive := isOpenCodeFormatStoragePath( - agent, storedPath, - ) || (storedPath == "" && storedHasStorageFingerprint) - if storedIsSQLiteVirtual { - storedIsStorageArchive = false - } - if isOpenCodeFormatSQLiteVirtualPath(agent, path) && - !storedIsStorageArchive { - return false - } - storedMsgs, err := store.GetAllMessages( - context.Background(), sessionID, - ) - if err != nil || len(storedMsgs) == 0 { - return false - } - // A changed storage fingerprint alone is not enough to - // preserve the archive. OpenCode legitimately rewrites - // live child files in place, so we only preserve when the - // newly parsed transcript also looks incomplete relative - // to what is already archived. - if storedHasStorageFingerprint && - hasOpenCodeFormatStorageFingerprint(agent, currentHash) && - !parser.OpenCodeStorageFingerprintMissing( - storedHash, currentHash, - ) { - return false - } - if storedIsStorageArchive && - isOpenCodeFormatSQLiteVirtualPath(agent, path) && - currentMtime != 0 && - storedMtime != 0 && - currentMtime <= storedMtime { - log.Printf( - "skip %s session %s: sqlite fallback is not newer than preserved storage archive", - agent, sessionID, - ) - return true - } - incomplete := false - if e.db.ArchiveContent().UsageOnly() { - _, comparableCurrentMsgs := e.db.ProjectSessionForStorage( - db.Session{}, currentMsgs, - ) - _, comparableStoredMsgs := e.db.ProjectSessionForStorage( - db.Session{}, storedMsgs, - ) - incomplete = openCodeUsageOnlyArchiveLooksIncomplete( - comparableCurrentMsgs, comparableStoredMsgs, - ) - } else { - incomplete = openCodeLegacyArchiveLooksIncomplete( - currentMsgs, storedMsgs, - ) - } - if incomplete { - if hasOpenCodeFormatStorageFingerprint(agent, storedHash) { - log.Printf( - "skip %s session %s: storage fingerprint changed but update looks incomplete relative to archive", - agent, sessionID, - ) - } else { - log.Printf( - "skip %s session %s: storage update looks incomplete relative to legacy archive", - agent, sessionID, - ) - } - return true - } - return false -} - func isOpenCodeFormatStorageAgent(agent parser.AgentType) bool { - return agent == parser.AgentOpenCode || - agent == parser.AgentKilo || - agent == parser.AgentIcodemate || - agent == parser.AgentMiMoCode + return ingest.IsOpenCodeFormatStorageAgent(agent) } func openCodeFormatDBName(agent parser.AgentType) string { - switch agent { - case parser.AgentOpenCode: - return "opencode.db" - case parser.AgentKilo: - return "kilo.db" - case parser.AgentMiMoCode: - return "mimocode.db" - case parser.AgentIcodemate: - return "icodemate.db" - default: - return "" - } + return ingest.OpenCodeFormatDBName(agent) } func resolveOpenCodeFormatSource( @@ -19317,45 +18482,22 @@ func openCodeFormatSourceMtime( } } -// hasOpenCodeFormatStorageFingerprint reports whether hash is an -// OpenCode storage fingerprint. Kilo reuses OpenCode's storage format -// verbatim, so the same check applies to both agents. -func hasOpenCodeFormatStorageFingerprint( - agent parser.AgentType, hash string, -) bool { - return isOpenCodeFormatStorageAgent(agent) && - parser.HasOpenCodeStorageFingerprint(hash) -} - func isOpenCodeFormatStoragePath( agent parser.AgentType, path string, ) bool { - return strings.HasSuffix(path, ".json") && - !isOpenCodeFormatSQLiteVirtualPath(agent, path) + return ingest.IsOpenCodeFormatStoragePath(agent, path) } func isOpenCodeFormatContainerSource( agent parser.AgentType, path string, ) bool { - return isOpenCodeFormatStorageAgent(agent) && - (isOpenCodeFormatStoragePath(agent, path) || - isOpenCodeFormatSQLiteVirtualPath(agent, path)) + return ingest.IsOpenCodeFormatContainerSource(agent, path) } func isOpenCodeFormatSQLiteVirtualPath( agent parser.AgentType, path string, ) bool { - if !isOpenCodeFormatStorageAgent(agent) { - return false - } - if agent == parser.AgentOpenCode { - _, _, ok := parser.ParseOpenCodeSQLiteVirtualPath(path) - return ok - } - _, _, ok := parser.ParseVirtualSourcePathForBase( - path, openCodeFormatDBName(agent), - ) - return ok + return ingest.IsOpenCodeFormatSQLiteVirtualPath(agent, path) } func derefString(s *string) string { @@ -19365,172 +18507,16 @@ func derefString(s *string) string { return *s } -func derefInt64(v *int64) int64 { - if v == nil { - return 0 - } - return *v -} - func openCodeLegacyArchiveLooksIncomplete( parsed, stored []db.Message, ) bool { - if parsed == nil { - return len(stored) > 0 - } - if len(parsed) < len(stored) { - return true - } - for i := range stored { - if openCodeMessageLooksIncomplete( - parsed[i], stored[i], - ) { - return true - } - } - return false + return ingest.OpenCodeLegacyArchiveLooksIncomplete(parsed, stored) } func openCodeUsageOnlyArchiveLooksIncomplete( parsed, stored []db.Message, ) bool { - if parsed == nil { - return len(stored) > 0 - } - if len(parsed) < len(stored) { - return true - } - parsedByIdentity := make( - map[openCodeMessageIdentity]db.Message, len(parsed), - ) - for _, message := range parsed { - parsedByIdentity[openCodeMessageStorageIdentity(message)] = message - // Older archives have no source UUID. Index the ordinal/role as - // well so those rows can adopt the source identity on replacement. - // Stored rows with UUIDs still require that exact UUID to match. - parsedByIdentity[openCodeMessageIdentity{ - ordinal: message.Ordinal, role: message.Role, - }] = message - } - for _, storedMessage := range stored { - parsedMessage, ok := parsedByIdentity[openCodeMessageStorageIdentity(storedMessage)] - if !ok || openCodeUsageOnlyMessageLooksIncomplete( - parsedMessage, storedMessage, - ) { - return true - } - } - return false -} - -type openCodeMessageIdentity struct { - sourceUUID string - ordinal int - role string -} - -func openCodeMessageStorageIdentity(message db.Message) openCodeMessageIdentity { - if message.SourceUUID != "" { - return openCodeMessageIdentity{sourceUUID: message.SourceUUID} - } - return openCodeMessageIdentity{ - ordinal: message.Ordinal, - role: message.Role, - } -} - -func openCodeMessageLooksIncomplete( - parsed, stored db.Message, -) bool { - if parsed.Ordinal != stored.Ordinal || - parsed.Role != stored.Role { - return false - } - if sanitizedMessageContentLength(parsed) < - sanitizedMessageContentLength(stored) { - return true - } - if parsed.HasThinking != stored.HasThinking && - stored.HasThinking { - return true - } - if stored.HasOutputTokens && - (!parsed.HasOutputTokens || - parsed.OutputTokens < stored.OutputTokens) { - return true - } - if stored.HasContextTokens && - (!parsed.HasContextTokens || - parsed.ContextTokens < stored.ContextTokens) { - return true - } - if len(parsed.ToolCalls) < len(stored.ToolCalls) { - return true - } - return countToolResultEvents(parsed.ToolCalls) < - countToolResultEvents(stored.ToolCalls) -} - -func openCodeUsageOnlyMessageLooksIncomplete( - parsed, stored db.Message, -) bool { - if parsed.Role != stored.Role { - return false - } - if openCodeUsageLooksIncomplete(parsed, stored) { - return true - } - // Usage rows keep only delegation calls, so every stored call must - // still be present with the same delegated session; a replacement - // call with the same count would otherwise drop a subagent link. - parsedLinks := make(map[string]string, len(parsed.ToolCalls)) - for _, call := range parsed.ToolCalls { - parsedLinks[call.ToolUseID] = call.SubagentSessionID - } - for _, call := range stored.ToolCalls { - child, ok := parsedLinks[call.ToolUseID] - if !ok || child != call.SubagentSessionID { - return true - } - } - return false -} - -func openCodeUsageLooksIncomplete(parsed, stored db.Message) bool { - if stored.Model != "" && parsed.Model == "" { - return true - } - if len(stored.TokenUsage) == 0 { - return false - } - if len(parsed.TokenUsage) == 0 { - return true - } - parsedUsage := usagefacts.ParseTokenUsage(string(parsed.TokenUsage)) - storedUsage := usagefacts.ParseTokenUsage(string(stored.TokenUsage)) - return parsedUsage.InputTokens < storedUsage.InputTokens || - parsedUsage.OutputTokens < storedUsage.OutputTokens || - parsedUsage.ReasoningTokens < storedUsage.ReasoningTokens || - parsedUsage.CacheCreationTokens < storedUsage.CacheCreationTokens || - parsedUsage.CacheCreation1hTokens < storedUsage.CacheCreation1hTokens || - parsedUsage.CacheReadTokens < storedUsage.CacheReadTokens || - parsedUsage.WebSearchRequests < storedUsage.WebSearchRequests -} - -func sanitizedMessageContentLength(msg db.Message) int { - sanitized := db.SanitizeUTF8(msg.Content) - if sanitized != msg.Content { - return len(sanitized) - } - return msg.ContentLength -} - -func countToolResultEvents(calls []db.ToolCall) int { - total := 0 - for _, call := range calls { - total += len(call.ResultEvents) - } - return total + return ingest.OpenCodeUsageOnlyArchiveLooksIncomplete(parsed, stored) } func (e *Engine) applyIDPrefixToSessionIDs(ids []string) []string { @@ -19559,6 +18545,10 @@ func (e *Engine) applyRemoteRewritesContext( p := applyIDPrefixToID(e.idPrefix, *s.ParentSessionID) s.ParentSessionID = &p } + if s.ParserParentSessionID != nil && *s.ParserParentSessionID != "" { + p := applyIDPrefixToID(e.idPrefix, *s.ParserParentSessionID) + s.ParserParentSessionID = &p + } if e.pathRewriter != nil && s.FilePath != nil { fp := e.pathRewriter(*s.FilePath) s.FilePath = &fp @@ -19604,59 +18594,7 @@ func toDBSession(pw pendingWrite) db.Session { func toDBSessionContext( ctx context.Context, pw pendingWrite, ) (db.Session, error) { - hasTotal, hasPeak, err := pw.sess.TokenCoverageContext(ctx, pw.msgs) - if err != nil { - return db.Session{}, err - } - s := db.Session{ - ID: pw.sess.ID, - Project: pw.sess.Project, - Machine: pw.sess.Machine, - MessageCount: pw.sess.MessageCount, - UserMessageCount: pw.sess.UserMessageCount, - ParentSessionID: strPtr(pw.sess.ParentSessionID), - RelationshipType: string(pw.sess.RelationshipType), - TotalOutputTokens: pw.sess.TotalOutputTokens, - PeakContextTokens: pw.sess.PeakContextTokens, - HasTotalOutputTokens: hasTotal, - HasPeakContextTokens: hasPeak, - Cwd: pw.sess.Cwd, - GitBranch: pw.sess.GitBranch, - SourceSessionID: pw.sess.SourceSessionID, - SourceVersion: pw.sess.SourceVersion, - TranscriptFidelity: pw.sess.TranscriptFidelity, - ParserMalformedLines: pw.sess.MalformedLines, - IsTruncated: pw.sess.IsTruncated, - TerminationStatus: strPtr(string(pw.sess.TerminationStatus)), - // data_version is intentionally left at the - // existing column default (0). UpsertSession does - // not persist this field; the caller bumps it via - // SetSessionDataVersion only after the message - // rewrite succeeds. - FilePath: strPtr(pw.sess.File.Path), - FileSize: int64Ptr(pw.sess.File.Size), - FileMtime: int64Ptr(pw.sess.File.Mtime), - NextOrdinal: nextParsedOrdinal(0, pw.msgs), - LastEntryUUID: strPtr(lastParsedSourceUUID("", pw.msgs)), - ClaudeLinearParse: pw.sess.ClaudeLinearParse, - FileInode: int64Ptr(pw.sess.File.Inode), - FileDevice: int64Ptr(pw.sess.File.Device), - FileHash: strPtr(pw.sess.File.Hash), - } - db.ApplyParsedSessionIdentity(&s, pw.sess) - if pw.sess.FirstMessage != "" { - s.FirstMessage = &pw.sess.FirstMessage - } - s.SessionName = db.ParsedSessionName(pw.sess) - s.PreserveSessionName = pw.sess.Agent == parser.AgentCodex && - !pw.sess.SessionNamePresent - if !pw.sess.StartedAt.IsZero() { - s.StartedAt = timeutil.Ptr(pw.sess.StartedAt) - } - if !pw.sess.EndedAt.IsZero() { - s.EndedAt = timeutil.Ptr(pw.sess.EndedAt) - } - return s, ctx.Err() + return ingest.ConvertSessionContext(ctx, pw.sess, pw.msgs) } // toDBMessages converts parsed messages to db.Message rows @@ -19669,60 +18607,9 @@ func toDBMessages(pw pendingWrite, blocked map[string]bool) []db.Message { func toDBMessagesContext( ctx context.Context, pw pendingWrite, blocked map[string]bool, ) ([]db.Message, error) { - msgs := make([]db.Message, len(pw.msgs)) - for i, m := range pw.msgs { - if err := ctx.Err(); err != nil { - return nil, err - } - hasCtx, hasOut := m.TokenPresence() - toolCalls, err := convertToolCallsContext(ctx, pw.sess.ID, m.ToolCalls) - if err != nil { - return nil, err - } - if isCodexFormatAgent(pw.sess.Agent) { - for j := range toolCalls { - for k := range toolCalls[j].ResultEvents { - db.PrepareToolResultEvent(&toolCalls[j].ResultEvents[k]) - } - } - } - toolResults, err := convertToolResultsContext(ctx, m.ToolResults) - if err != nil { - return nil, err - } - msgs[i] = db.Message{ - SessionID: pw.sess.ID, - Ordinal: m.Ordinal, - Role: string(m.Role), - Content: m.Content, - ThinkingText: m.ThinkingText, - Timestamp: timeutil.Format(m.Timestamp), - HasThinking: m.HasThinking, - HasToolUse: m.HasToolUse, - ContentLength: m.ContentLength, - IsSystem: m.IsSystem, - Model: m.Model, - ReasoningEffort: m.ReasoningEffort, - ProviderID: m.ProviderID, - TokenUsage: m.TokenUsage, - ContextTokens: m.ContextTokens, - OutputTokens: m.OutputTokens, - HasContextTokens: hasCtx, - HasOutputTokens: hasOut, - ClaudeMessageID: m.ClaudeMessageID, - ClaudeRequestID: m.ClaudeRequestID, - SourceType: m.SourceType, - SourceSubtype: m.SourceSubtype, - PromptSource: m.PromptSource, - SourceUUID: m.SourceUUID, - SourceParentUUID: m.SourceParentUUID, - IsSidechain: m.IsSidechain, - IsCompactBoundary: m.IsCompactBoundary, - ToolCalls: toolCalls, - ToolResults: toolResults, - } - } - return pairAndFilterContext(ctx, msgs, blocked) + return ingest.ConvertMessagesContext( + ctx, pw.sess.ID, pw.sess.Agent, pw.msgs, blocked, + ) } // toDBUsageEvents converts parser usage events for one session. @@ -19742,57 +18629,7 @@ func toDBUsageEvents( func toDBUsageEventsContext( ctx context.Context, sessionID string, events []parser.ParsedUsageEvent, ) ([]db.UsageEvent, validationStats, error) { - out := make([]db.UsageEvent, 0, len(events)) - for _, ev := range events { - if err := ctx.Err(); err != nil { - return nil, validationStats{}, err - } - out = append(out, db.UsageEvent{ - SessionID: sessionID, - MessageOrdinal: ev.MessageOrdinal, - Source: ev.Source, - Model: ev.Model, - ProviderID: ev.ProviderID, - InputTokens: ev.InputTokens, - OutputTokens: ev.OutputTokens, - CacheCreationInputTokens: ev.CacheCreationInputTokens, - CacheReadInputTokens: ev.CacheReadInputTokens, - ReasoningTokens: ev.ReasoningTokens, - Cost: ev.Cost, - CostStatus: ev.CostStatus, - CostSource: ev.CostSource, - OccurredAt: ev.OccurredAt, - DedupKey: ev.DedupKey, - }) - } - // Route usage events through the central validation/sanitization - // pass so they get the same treatment as messages and sessions at - // every call site. - stats, err := validateAndSanitizeContext(ctx, nil, nil, out) - return out, stats, err -} - -// usageEventsForWrite converts usage events for a session about to be -// written and records the central-validation fix counts in the per-run -// anomaly accumulator for the sync summary. -func (e *Engine) usageEventsForWrite( - sessionID string, events []parser.ParsedUsageEvent, -) []db.UsageEvent { - out, _ := e.usageEventsForWriteContext( - context.Background(), sessionID, events, - ) - return out -} - -func (e *Engine) usageEventsForWriteContext( - ctx context.Context, sessionID string, events []parser.ParsedUsageEvent, -) ([]db.UsageEvent, error) { - out, vs, err := toDBUsageEventsContext(ctx, sessionID, events) - if err != nil { - return nil, err - } - e.anomalies.recordSanitize(vs) - return out, nil + return ingest.ConvertUsageEventsContext(ctx, sessionID, events) } // postFilterCounts returns the total and user message counts @@ -19851,21 +18688,13 @@ func countUserMsgs(msgs []parser.ParsedMessage) int { func nextParsedOrdinal( current int, msgs []parser.ParsedMessage, ) int { - if len(msgs) == 0 { - return current - } - return msgs[len(msgs)-1].Ordinal + 1 + return ingest.NextParsedOrdinal(current, msgs) } func lastParsedSourceUUID( current string, msgs []parser.ParsedMessage, ) string { - for _, v := range slices.Backward(msgs) { - if v.SourceUUID != "" { - return v.SourceUUID - } - } - return current + return ingest.LastParsedSourceUUID(current, msgs) } // FindSourceFile locates the original source file for a @@ -21079,37 +19908,7 @@ func convertToolCalls( func convertToolCallsContext( ctx context.Context, sessionID string, parsed []parser.ParsedToolCall, ) ([]db.ToolCall, error) { - if len(parsed) == 0 { - return nil, ctx.Err() - } - calls := make([]db.ToolCall, len(parsed)) - for i, tc := range parsed { - if err := ctx.Err(); err != nil { - return nil, err - } - filePath := tc.FilePath - if filePath == "" { - filePath = parser.ResolveFilePathFromJSON(tc.InputJSON) - } - resultEvents, err := convertToolResultEventsContext(ctx, tc.ResultEvents) - if err != nil { - return nil, err - } - calls[i] = db.ToolCall{ - SessionID: sessionID, - ToolName: tc.ToolName, - Category: tc.Category, - ToolUseID: tc.ToolUseID, - InputJSON: tc.InputJSON, - Rendering: tc.Rendering, - FilePath: filePath, - CallIndex: i, - SkillName: tc.SkillName, - SubagentSessionID: tc.SubagentSessionID, - ResultEvents: resultEvents, - } - } - return calls, ctx.Err() + return ingest.ConvertToolCallsContext(ctx, sessionID, parsed) } // prepareCodexResultIdentities runs in parse workers before their payloads @@ -21141,50 +19940,7 @@ func prepareCodexResultIdentities( func convertToolResultEventsContext( ctx context.Context, parsed []parser.ParsedToolResultEvent, ) ([]db.ToolResultEvent, error) { - if len(parsed) == 0 { - return nil, ctx.Err() - } - events := make([]db.ToolResultEvent, len(parsed)) - for i, ev := range parsed { - if err := ctx.Err(); err != nil { - return nil, err - } - events[i] = db.ToolResultEvent{ - RawContentDigest: ev.RawContentDigest, - ToolUseID: ev.ToolUseID, - AgentID: ev.AgentID, - SubagentSessionID: ev.SubagentSessionID, - Source: ev.Source, - Status: ev.Status, - Content: ev.Content, - ContentLength: len(ev.Content), - Timestamp: timeutil.Format(ev.Timestamp), - EventIndex: i, - } - } - return events, ctx.Err() -} - -// convertToolResults maps parsed tool results to db.ToolResult -// structs for use in pairing before DB insert. -func convertToolResultsContext( - ctx context.Context, parsed []parser.ParsedToolResult, -) ([]db.ToolResult, error) { - if len(parsed) == 0 { - return nil, ctx.Err() - } - results := make([]db.ToolResult, len(parsed)) - for i, tr := range parsed { - if err := ctx.Err(); err != nil { - return nil, err - } - results[i] = db.ToolResult{ - ToolUseID: tr.ToolUseID, - ContentLength: tr.ContentLength, - ContentRaw: tr.ContentRaw, - } - } - return results, ctx.Err() + return ingest.ConvertToolResultEventsContext(ctx, parsed) } // pairAndFilter pairs tool results with their corresponding @@ -21200,27 +19956,7 @@ func pairAndFilter(msgs []db.Message, blocked map[string]bool) []db.Message { func pairAndFilterContext( ctx context.Context, msgs []db.Message, blocked map[string]bool, ) ([]db.Message, error) { - if err := pairToolResultsContext(ctx, msgs, blocked); err != nil { - return nil, err - } - if err := pairToolResultEventSummariesContext( - ctx, msgs, blocked, - ); err != nil { - return nil, err - } - filtered := msgs[:0] - for _, m := range msgs { - if err := ctx.Err(); err != nil { - return nil, err - } - if m.Role == "user" && - len(m.ToolResults) > 0 && - strings.TrimSpace(m.Content) == "" { - continue - } - filtered = append(filtered, m) - } - return filtered, ctx.Err() + return ingest.PairAndFilterContext(ctx, msgs, blocked) } // pairToolResults matches tool_result content to their @@ -21233,47 +19969,7 @@ func pairToolResults(msgs []db.Message, blocked map[string]bool) { func pairToolResultsContext( ctx context.Context, msgs []db.Message, blocked map[string]bool, ) error { - idx := make(map[string]*db.ToolCall) - for i := range msgs { - if err := ctx.Err(); err != nil { - return err - } - for j := range msgs[i].ToolCalls { - if err := ctx.Err(); err != nil { - return err - } - tc := &msgs[i].ToolCalls[j] - if tc.ToolUseID != "" { - idx[tc.ToolUseID] = tc - } - } - } - if len(idx) == 0 { - return ctx.Err() - } - for _, m := range msgs { - if err := ctx.Err(); err != nil { - return err - } - for _, tr := range m.ToolResults { - if err := ctx.Err(); err != nil { - return err - } - if tc, ok := idx[tr.ToolUseID]; ok { - // A withheld result keeps the parser's size; a stored one - // is measured, matching the archive's write rule so change - // detection never sees a parser-side rounding difference. - tc.ResultContentLength = tr.ContentLength - if !blocked[tc.Category] { - tc.ResultContent = parser.DecodeContent(tr.ContentRaw) - tc.ResultContentLength = db.ResolveResultContentLength( - tc.ResultContent, tr.ContentLength, - ) - } - } - } - } - return ctx.Err() + return ingest.PairToolResultsContext(ctx, msgs, blocked) } func pairToolResultEventSummaries( @@ -21287,98 +19983,7 @@ func pairToolResultEventSummaries( func pairToolResultEventSummariesContext( ctx context.Context, msgs []db.Message, blocked map[string]bool, ) error { - for i := range msgs { - if err := ctx.Err(); err != nil { - return err - } - for j := range msgs[i].ToolCalls { - if err := ctx.Err(); err != nil { - return err - } - tc := &msgs[i].ToolCalls[j] - if len(tc.ResultEvents) == 0 { - continue - } - summary, err := summarizeToolResultEventsContext( - ctx, tc.ResultEvents, - ) - if err != nil { - return err - } - tc.ResultContentLength = len(summary) - if blocked[tc.Category] { - tc.ResultContent = "" - for k := range tc.ResultEvents { - tc.ResultEvents[k].Content = "" - } - continue - } - tc.ResultContent = summary - } - } - return ctx.Err() -} - -func summarizeToolResultEventsContext( - ctx context.Context, events []db.ToolResultEvent, -) (string, error) { - if len(events) == 0 { - return "", ctx.Err() - } - type agentSummary struct { - order int - content string - } - latestByAgent := map[string]agentSummary{} - orderedAgents := make([]string, 0, len(events)) - lastAnon := "" - allHaveAgentID := true - for _, ev := range events { - if err := ctx.Err(); err != nil { - return "", err - } - if strings.TrimSpace(ev.Content) == "" { - continue - } - agentID := strings.TrimSpace(ev.AgentID) - if agentID == "" { - allHaveAgentID = false - lastAnon = ev.Content - continue - } - if _, ok := latestByAgent[agentID]; !ok { - latestByAgent[agentID] = agentSummary{ - order: len(orderedAgents), - content: ev.Content, - } - orderedAgents = append(orderedAgents, agentID) - continue - } - entry := latestByAgent[agentID] - entry.content = ev.Content - latestByAgent[agentID] = entry - } - if len(latestByAgent) <= 1 { - if len(latestByAgent) == 1 { - summary := latestByAgent[orderedAgents[0]].content - if lastAnon != "" { - return summary + "\n\n" + lastAnon, ctx.Err() - } - return summary, ctx.Err() - } - return lastAnon, ctx.Err() - } - parts := make([]string, 0, len(orderedAgents)) - for _, agentID := range orderedAgents { - if err := ctx.Err(); err != nil { - return "", err - } - parts = append(parts, agentID+":\n"+latestByAgent[agentID].content) - } - if !allHaveAgentID && lastAnon != "" { - parts = append(parts, lastAnon) - } - return strings.Join(parts, "\n\n"), ctx.Err() + return ingest.PairToolResultEventSummariesContext(ctx, msgs, blocked) } // emit fires a refresh event if an emitter is wired. Safe to call diff --git a/internal/sync/history_reconcile_test.go b/internal/sync/history_reconcile_test.go new file mode 100644 index 0000000000..f889aa5a58 --- /dev/null +++ b/internal/sync/history_reconcile_test.go @@ -0,0 +1,54 @@ +package sync + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/parser" +) + +type providerHistoryReadSpy struct { + db.Store + getSessionCalls int + getMessageCalls int +} + +func (s *providerHistoryReadSpy) GetSessionFull( + context.Context, string, +) (*db.Session, error) { + s.getSessionCalls++ + return nil, nil +} + +func (s *providerHistoryReadSpy) GetAllMessages( + context.Context, string, +) ([]db.Message, error) { + s.getMessageCalls++ + return nil, nil +} + +func TestReconcileProviderHistorySkipsPriorReadForICodeMateCLI(t *testing.T) { + archive := &providerHistoryReadSpy{} + engine := &Engine{db: openTestDB(t), archiveStore: archive} + candidate := ingest.Candidate{ + Session: db.Session{ + ID: "icodemate:session-1", Agent: string(parser.AgentIcodemate), + }, + Parsed: parser.ParseResult{Session: parser.ParsedSession{ + ID: "session-1", Agent: parser.AgentIcodemate, + File: parser.FileInfo{Path: "/synthetic/project/session-1.jsonl"}, + }}, + } + + result, err := engine.reconcileProviderHistoryContext( + context.Background(), candidate, + ) + require.NoError(t, err) + assert.Equal(t, ingest.HistoryReplace, result.Action) + assert.Zero(t, archive.getSessionCalls) + assert.Zero(t, archive.getMessageCalls) +} diff --git a/internal/sync/secret_scan.go b/internal/sync/secret_scan.go index d3f0b94b72..889e0eb3f8 100644 --- a/internal/sync/secret_scan.go +++ b/internal/sync/secret_scan.go @@ -4,19 +4,19 @@ import ( "sync/atomic" "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/secrets" "go.kenn.io/agentsview/internal/signals" ) -// secretScanBytes counts the content bytes passed to the secret scanner. -// Tests use deltas of it to gate the incremental path: a maintained delta -// must scan no more than the delta's own content. +// secretScanBytes tracks incremental-only scans; shared full scans keep their +// counter in ingest. var secretScanBytes atomic.Int64 // SecretScanBytes returns the total scanned byte count so far. Monotonic. func SecretScanBytes() int64 { - return secretScanBytes.Load() + return ingest.SecretScanBytes() + secretScanBytes.Load() } // computeSignalsAndSecrets computes a session's signal update and its secret @@ -32,11 +32,7 @@ func SecretScanBytes() int64 { func computeSignalsAndSecrets( s db.Session, msgs []db.Message, ) (db.SessionSignalUpdate, []db.SecretFinding) { - update := computeSignalsFromMessages(s, msgs) - findings, leak := scanSecretsFromMessages(s, msgs, secrets.ScanDefinite) - update.SecretLeakCount = leak - update.SecretsRulesVersion = secrets.DefiniteRulesVersion() - return update, findings + return ingest.ComputeSignalsAndSecrets(s, msgs) } // computeSignalsAndSecretsWithContentFailures is the staged streaming @@ -47,11 +43,9 @@ func computeSignalsAndSecrets( func computeSignalsAndSecretsWithContentFailures( s db.Session, msgs []db.Message, failures map[string]bool, ) (db.SessionSignalUpdate, []db.SecretFinding) { - update := computeSignalsFromMessagesWithContentFailures(s, msgs, failures) - findings, leak := scanSecretsFromMessages(s, msgs, secrets.ScanDefinite) - update.SecretLeakCount = leak - update.SecretsRulesVersion = secrets.DefiniteRulesVersion() - return update, findings + return ingest.ComputeSignalsAndSecretsWithContentFailures( + s, msgs, failures, + ) } // computeFullSignalsAndSecrets prepares all derived state for a full content @@ -86,6 +80,32 @@ func computeFullSignalsAndSecrets( return update, findings, nil } +// attachFullSignalState adds the SQLite-only incremental seed to an aggregate +// update already computed by shared ingestion. It does not rescan messages or +// recompute aggregate signals. +func (e *Engine) attachFullSignalState( + s db.Session, msgs []db.Message, update db.SessionSignalUpdate, + failures map[string]bool, +) (db.SessionSignalUpdate, error) { + if !isCodexFormatAgent(parser.AgentType(s.Agent)) { + return update, nil + } + rows := extractToolCallRows(msgs) + patchToolCallRowsWithContentFailures(rows, msgs, failures) + for i := range rows { + if rows[i].EventStatus == "" { + rows[i].ContentFailure = signals.IsFailure(rows[i]) + rows[i].ContentFailureKnown = true + } + } + state, err := buildSignalStateFromRows(s.ID, msgs, rows, "") + if err != nil { + return db.SessionSignalUpdate{}, err + } + update.FullState = &state + return update, nil +} + // scanSecretsFromMessages detects secrets across a session's message content, // tool inputs, and canonical tool output (result events when present, else // result_content) using scan: secrets.ScanDefinite for the fast inline path, @@ -93,61 +113,9 @@ func computeFullSignalsAndSecrets( // count of definite findings (the secret_leak_count signal). Pure: no DB // access. func scanSecretsFromMessages( - _ db.Session, msgs []db.Message, scan func(string) []secrets.Match, + s db.Session, msgs []db.Message, scan func(string) []secrets.Match, ) (findings []db.SecretFinding, definiteCount int) { - findings = make([]db.SecretFinding, 0) - add := func(sessionID, loc string, ord int, call, event *int, content string, matches []secrets.Match) { - secretScanBytes.Add(int64(len(content))) - for _, m := range matches { - findings = append(findings, db.SecretFinding{ - SessionID: sessionID, - RuleName: m.Rule, - Confidence: m.Confidence, - LocationKind: loc, - MessageOrdinal: ord, - CallIndex: call, - EventIndex: event, - MatchStart: m.Start, - MatchEnd: m.End, - MatchIndex: m.Index, - RedactedMatch: m.Redacted, - // The incremental persist path (applySignalDeltaTx) inserts - // f.RulesVersion verbatim, unlike replaceSecretFindingsTx - // which overrides it; stamp it here so inline findings are - // visible to current-version listings. - RulesVersion: secrets.DefiniteRulesVersion(), - }) - if m.Confidence == secrets.ConfidenceDefinite { - definiteCount++ - } - } - } - for _, msg := range msgs { - add(msg.SessionID, "message", msg.Ordinal, nil, nil, - msg.Content, scan(msg.Content)) - for ci := range msg.ToolCalls { - tc := msg.ToolCalls[ci] - callIdx := ci - add(msg.SessionID, "tool_input", msg.Ordinal, &callIdx, nil, - tc.InputJSON, scan(tc.InputJSON)) - if len(tc.ResultEvents) > 0 { - for ei := range tc.ResultEvents { - // Store the slice position, which is what the persistence - // layer (resolveToolResultEvents) writes as event_index. - // SecretFindingSource reads findings back through the same - // normalized value, so --reveal can re-locate the source. - evIdx := ei - add(msg.SessionID, "tool_result_event", msg.Ordinal, - &callIdx, &evIdx, tc.ResultEvents[ei].Content, - scan(tc.ResultEvents[ei].Content)) - } - } else { - add(msg.SessionID, "tool_result", msg.Ordinal, &callIdx, nil, - tc.ResultContent, scan(tc.ResultContent)) - } - } - } - return findings, definiteCount + return ingest.ScanSecretsFromMessages(s, msgs, scan) } // computeSignalsAndSecretsForStorage computes signals and findings from the diff --git a/internal/sync/signal_compute.go b/internal/sync/signal_compute.go index e50298f973..c24ee3428e 100644 --- a/internal/sync/signal_compute.go +++ b/internal/sync/signal_compute.go @@ -1,317 +1,51 @@ package sync import ( - "slices" - "time" - "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/ingest" "go.kenn.io/agentsview/internal/signals" ) -// computeSignalsFromMessages produces a SessionSignalUpdate from -// in-memory session metadata and messages. Pure function with no -// DB access — the caller already holds everything we need. -// -// Used by both the live write paths (writeBatch, writeSessionFull, -// writeIncremental) and the legacy backfill path (RecomputeSignals, -// which reads msgs from the DB once and then calls this). +// Compatibility wrappers keep incremental maintenance and existing package +// tests on the shared normalized computation. func computeSignalsFromMessages( - sess db.Session, msgs []db.Message, -) db.SessionSignalUpdate { - return computeSignalsFromToolRows(sess, msgs, extractToolCallRows(msgs)) -} - -// computeSignalsFromMessagesWithContentFailures is the staged streaming -// variant: tool rows whose placeholder content hides a pre-computed -// content-failure verdict get that verdict stamped before the signal pass, -// so content-driven tool-health signals match the collecting path byte for -// byte. -func computeSignalsFromMessagesWithContentFailures( - sess db.Session, msgs []db.Message, failures map[string]bool, + session db.Session, messages []db.Message, ) db.SessionSignalUpdate { - toolRows := extractToolCallRows(msgs) - patchToolCallRowsWithContentFailures(toolRows, msgs, failures) - return computeSignalsFromToolRows(sess, msgs, toolRows) + return ingest.ComputeSignalsFromMessages(session, messages) } -// patchToolCallRowsWithContentFailures stamps pre-computed content-failure -// verdicts onto rows whose last event status is empty (status-driven -// verdicts win). toolRows must be the output of extractToolCallRows over -// the same msgs slice, so the walk order is identical. func patchToolCallRowsWithContentFailures( - toolRows []signals.ToolCallRow, - msgs []db.Message, - failures map[string]bool, + rows []signals.ToolCallRow, messages []db.Message, failures map[string]bool, ) { - if len(failures) == 0 { - return - } - idx := 0 - callOccurrences := make(map[string]int) - for _, m := range msgs { - for _, tc := range m.ToolCalls { - if idx >= len(toolRows) { - return - } - failed := false - if tc.ToolUseID != "" { - occurrence := callOccurrences[tc.ToolUseID] - callOccurrences[tc.ToolUseID] = occurrence + 1 - failed = failures[db.StagedToolCallKey( - tc.ToolUseID, occurrence, - )] - } - if failed && toolRows[idx].EventStatus == "" { - toolRows[idx].ContentFailure = true - } - idx++ - } - } + ingest.PatchToolCallRowsWithContentFailures(rows, messages, failures) } func computeSignalsFromToolRows( - sess db.Session, msgs []db.Message, toolRows []signals.ToolCallRow, + session db.Session, messages []db.Message, rows []signals.ToolCallRow, ) db.SessionSignalUpdate { - heuristics := signals.AnalyzeHeuristics(signals.HeuristicInput{ - Messages: extractHeuristicMessages(msgs), - ToolRows: toolRows, - }) - ctxTokens := extractContextTokens(msgs) - boundaries := extractCompactBoundaryOrdinals(msgs) - model := extractMostCommonModel(msgs) - lastRole, lastContent := extractLastMessageRole(msgs) - - toolHealth := signals.ComputeToolHealth(toolRows) - ctxPressure := signals.ComputeContextPressure( - ctxTokens, sess.PeakContextTokens, model, - ) - - // Prefer explicit boundary count when available; fall back - // to the token-drop heuristic for sessions without - // compact-boundary messages. - compactionCount := ctxPressure.CompactionCount - if len(boundaries) > 0 { - compactionCount = len(boundaries) - } - - midTaskCalls := make( - []signals.ToolCallOrdinal, 0, len(toolRows), - ) - for _, t := range toolRows { - midTaskCalls = append(midTaskCalls, - signals.ToolCallOrdinal{ - MessageOrdinal: t.MessageOrdinal, - ToolName: t.ToolName, - }) - } - midTaskCount := signals.CountMidTaskCompactions( - boundaries, midTaskCalls, - ) - - finalStreak := computeFinalStreak(toolRows) - - var lastActivity time.Time - if sess.EndedAt != nil { - lastActivity, _ = time.Parse( - time.RFC3339Nano, *sess.EndedAt, - ) - } - - outcomeResult := signals.ClassifyOutcome(signals.OutcomeInput{ - IsAutomated: sess.IsAutomated, - MessageCount: sess.MessageCount, - EndedWithRole: lastRole, - FinalFailureStreak: finalStreak, - LastAssistantText: lastContent, - LastActivity: lastActivity, - }) - - hasContextData := sess.HasPeakContextTokens - if !hasContextData { - for _, t := range ctxTokens { - if t.HasContextTokens { - hasContextData = true - break - } - } - } - - scoreResult := signals.ComputeHealthScore(signals.ScoreInput{ - Outcome: outcomeResult.Outcome, - OutcomeConfidence: outcomeResult.Confidence, - HasToolCalls: len(toolRows) > 0, - FailureSignalCount: toolHealth.FailureSignalCount, - RetryCount: toolHealth.RetryCount, - EditChurnCount: toolHealth.EditChurnCount, - ConsecutiveFailMax: toolHealth.ConsecutiveFailureMax, - HasContextData: hasContextData, - CompactionCount: compactionCount, - MidTaskCompactionCount: midTaskCount, - PressureMax: ctxPressure.PressureMax, - Heuristics: heuristics, - }) - - var pendingSince *string - if outcomeResult.IsRecent { - now := time.Now().UTC().Format(time.RFC3339) - pendingSince = &now - } - - var healthGrade *string - if scoreResult.Grade != "" { - healthGrade = &scoreResult.Grade - } - - return db.SessionSignalUpdate{ - ToolFailureSignalCount: toolHealth.FailureSignalCount, - ToolRetryCount: toolHealth.RetryCount, - EditChurnCount: toolHealth.EditChurnCount, - ConsecutiveFailureMax: toolHealth.ConsecutiveFailureMax, - Outcome: outcomeResult.Outcome, - OutcomeConfidence: outcomeResult.Confidence, - EndedWithRole: lastRole, - FinalFailureStreak: finalStreak, - SignalsPendingSince: pendingSince, - CompactionCount: compactionCount, - MidTaskCompactionCount: midTaskCount, - ContextPressureMax: ctxPressure.PressureMax, - HealthScore: scoreResult.Score, - HealthGrade: healthGrade, - HasToolCalls: len(toolRows) > 0, - HasContextData: hasContextData, - QualitySignals: db.QualitySignals{ - Version: db.CurrentQualitySignalVersion, - ShortPromptCount: heuristics.ShortPromptCount, - UnstructuredStart: heuristics.UnstructuredStart, - MissingSuccessCriteriaCount: heuristics. - MissingSuccessCriteriaCount, - MissingVerificationCount: heuristics. - MissingVerificationCount, - DuplicatePromptCount: heuristics.DuplicatePromptCount, - NoCodeContextCount: heuristics.NoCodeContextCount, - RunawayToolLoopCount: heuristics.RunawayToolLoopCount, - }, - } + return ingest.ComputeSignalsFromToolRows(session, messages, rows) } -func extractHeuristicMessages( - msgs []db.Message, -) []signals.HeuristicMessage { - rows := make([]signals.HeuristicMessage, 0, len(msgs)) - for _, m := range msgs { - rows = append(rows, signals.HeuristicMessage{ - Role: m.Role, - SourceSubtype: m.SourceSubtype, - Content: m.Content, - IsSystem: m.IsSystem, - Ordinal: m.Ordinal, - Timestamp: m.Timestamp, - }) - } - return rows +func extractHeuristicMessages(messages []db.Message) []signals.HeuristicMessage { + return ingest.ExtractHeuristicMessages(messages) } -// extractToolCallRows builds signal inputs from in-memory tool -// calls. CallIndex is the call's position within its message. -// EventStatus is the status of the latest result event (events -// are stored in event_index order, so the last one wins). -func extractToolCallRows( - msgs []db.Message, -) []signals.ToolCallRow { - rows := make([]signals.ToolCallRow, 0) - for _, m := range msgs { - for callIdx, tc := range m.ToolCalls { - status := "" - if n := len(tc.ResultEvents); n > 0 { - status = tc.ResultEvents[n-1].Status - } - rows = append(rows, signals.ToolCallRow{ - ToolName: tc.ToolName, - Category: tc.Category, - InputJSON: tc.InputJSON, - ResultContent: tc.ResultContent, - MessageOrdinal: m.Ordinal, - CallIndex: callIdx, - EventStatus: status, - }) - } - } - return rows +func extractToolCallRows(messages []db.Message) []signals.ToolCallRow { + return ingest.ExtractToolCallRows(messages) } -// extractContextTokens returns context-token measurements for -// assistant messages in order. -func extractContextTokens( - msgs []db.Message, -) []signals.ContextTokenRow { - var rows []signals.ContextTokenRow - for _, m := range msgs { - if m.Role != "assistant" { - continue - } - rows = append(rows, signals.ContextTokenRow{ - ContextTokens: m.ContextTokens, - HasContextTokens: m.HasContextTokens, - }) - } - return rows +func extractContextTokens(messages []db.Message) []signals.ContextTokenRow { + return ingest.ExtractContextTokens(messages) } -// extractCompactBoundaryOrdinals returns ordinals of explicit -// compact-boundary messages in ascending order. -func extractCompactBoundaryOrdinals(msgs []db.Message) []int { - var ords []int - for _, m := range msgs { - if m.IsCompactBoundary { - ords = append(ords, m.Ordinal) - } - } - return ords +func extractCompactBoundaryOrdinals(messages []db.Message) []int { + return ingest.ExtractCompactBoundaryOrdinals(messages) } -// extractMostCommonModel returns the assistant model name that -// appears most often. Ties are broken by the model that appears -// first chronologically — matches SQLite's GROUP BY iteration -// order closely enough that legacy and live computations agree -// on real sessions (every observed session has a clear majority -// model). -func extractMostCommonModel(msgs []db.Message) string { - counts := map[string]int{} - firstSeen := map[string]int{} - for i, m := range msgs { - if m.Role != "assistant" || m.Model == "" { - continue - } - counts[m.Model]++ - if _, ok := firstSeen[m.Model]; !ok { - firstSeen[m.Model] = i - } - } - var best string - bestCount := -1 - for model, n := range counts { - switch { - case n > bestCount: - best, bestCount = model, n - case n == bestCount && firstSeen[model] < firstSeen[best]: - best = model - } - } - return best +func extractMostCommonModel(messages []db.Message) string { + return ingest.ExtractMostCommonModel(messages) } -// extractLastMessageRole returns the role and content of the -// last non-system, non-tool-result message. Empty strings if none. -func extractLastMessageRole( - msgs []db.Message, -) (role, content string) { - if msgs == nil { - return "", "" - } - for _, v := range slices.Backward(msgs) { - if !v.IsSystem && v.SourceSubtype != "tool_result" { - return v.Role, v.Content - } - } - return "", "" +func extractLastMessageRole(messages []db.Message) (role, content string) { + return ingest.ExtractLastMessageRole(messages) }