feat: derive hosted raw captures into PostgreSQL sessions

.roborev.toml

@@ -1,10 +1,12 @@
 review_guidelines = """
-agentsview is a single-user developer tool. Default mode binds to
+agentsview defaults to a single-user developer tool. Local mode binds to
 127.0.0.1. Optional managed Caddy proxy mode allows LAN access while
 keeping the backend on loopback. Optional remote access mode binds to
 0.0.0.0 with bearer-token auth for use over secure tunnels (Tailscale,
-SSH, reverse proxy with TLS). Not designed for multi-user or
-internet-facing deployment.
+SSH, reverse proxy with TLS). These legacy modes are not designed for
+request-multiplexed multi-user deployment. Opt-in hosted raw processing has
+a separate threat model: one configured tenant, schema, restricted role and
+server instance, with authenticated uploads treated as untrusted input.
 
 Key assumptions reviewers MUST account for:
 
@@ -15,7 +17,9 @@ Key assumptions reviewers MUST account for:
    bearer token is required for all API requests (including localhost,
    to prevent bypass via reverse proxy). An auth token is
    auto-generated at startup if missing. Do not flag missing auth on
-   local-only code paths. DO flag any path that lets the backend bind
+   local-only code paths. Hosted mode always requires authentication;
+   raw-sync routes use device credentials and scoped tokens separately
+   from the viewer bearer token. DO flag any path that lets the backend bind
    non-loopback in proxy mode, or missing subnet checks for
    non-loopback Caddy binds.
 
@@ -37,34 +41,41 @@ Key assumptions reviewers MUST account for:
 4. XSS: {@html renderMarkdown(...)} is safe — renderMarkdown()
    sanitizes via DOMPurify before returning HTML.
 
-5. RATE LIMITING: Single-user tool. Do not flag missing rate limits
-   or concurrency caps.
+5. RATE LIMITING: Do not flag missing rate limits or concurrency
+   caps in local/legacy single-user paths. Hosted raw ingestion and
+   derivation must bound body, materialization, output and worker resources.
 
 6. CORS: corsMiddleware requires matching Origin for mutating
    requests. Allowed origins = loopback variants + public_url +
    public_origins. In remote mode, authenticated requests (bearer
    token) allow the request origin. Do not flag as overly permissive
    unless origins outside the configured set are accepted.
 
-7. INPUT VALIDATION: Body size limits not required — backend is
-   loopback-only; in proxy mode Caddy subnet filtering limits
+7. INPUT VALIDATION: In local/legacy paths, body size limits are not
+   required — backend is loopback-only; in proxy mode Caddy subnet filtering limits
    clients to trusted hosts. In remote mode, bearer token gates
-   all API access.
+   all API access. Hosted raw authentication does not make source bytes
+   trusted; review bounded protocol and parser validation.
 
 8. SESSION DATA: Displaying session contents (tool args, commands,
    paths) is the tool's purpose. The user owns these files. Do not
-   flag as sensitive data exposure.
+   flag as sensitive data exposure within that user's local/legacy access
+   boundary. Hosted reads must preserve tenant and public-identity isolation.
 
-9. SUBPROCESS ENV: Agent CLI subprocesses intentionally inherit the
-   parent environment. Do not flag env var inheritance.
+9. SUBPROCESS ENV: Local insight/capture agent CLIs intentionally inherit
+   the parent environment. Hosted raw parser children require an explicit
+   minimal environment and pre-runtime inherited-descriptor closure.
 
-10. SESSION PARSING: Input files are from local agent CLIs, not
-    adversarial. Do not flag missing cycle detection, recursion
-    limits, or unreachable-node checks in DAG traversals.
+10. SESSION PARSING: Local/legacy input files are user-owned. Do not flag
+    missing cycle detection, recursion limits or unreachable-node checks
+    there. Hosted raw input is untrusted; review fail-closed filesystem,
+    network, process and resource isolation before parsing.
 
-11. WRITE ATOMICITY: Per-session transactions only. Full resync
-    recovers partial state. Do not flag non-atomic multi-session
-    writes.
+11. WRITE ATOMICITY: Local/legacy writes use per-session transactions;
+    full resync recovers partial state. Hosted publication must atomically
+    fence selection and lease generations, normalized rows, provenance,
+    public identity, curation, outbox and job outcome. Partial sources must
+    retain unresolved proof and retry finitely.
 
 12. TOCTOU ON LOCAL FILES: Files in ~/.agentsview/ are user-owned.
     An attacker with home-directory access already has equivalent
@@ -132,7 +143,11 @@ Key assumptions reviewers MUST account for:
     go.mod toolchain version before flagging unfamiliar language
     features.
 
-Do NOT flag issues that only apply to public-facing, multi-tenant,
-or internet-exposed services. Focus on bugs, logic errors, data
-corruption risks, and code quality issues.
+For local/legacy modes, do not impose requirements that apply only to
+public-facing multi-tenant services. For opt-in hosted raw processing,
+review its actual tenant/role/catalog boundary, immutable source provenance,
+source-selection and lease fences, bounded isolated parsing, public-ID
+ambiguity, curation survival and partial-source retry semantics. Do not
+assume request-multiplexed tenancy. Focus on bugs, data corruption, boundary
+violations and maintainability within each mode's supported contract.
 """

README.md

@@ -1,3 +1,7 @@
+---
+last_edited: 2026-09-11
+---
+
 # agentsview
 
 Browse, search, and track costs across all your AI coding agents. One binary, no
@@ -557,6 +561,11 @@ agentsview pg status --all     # show status for every configured PG target
 agentsview pg serve            # serve web UI from the default PG target (read-only)
 ```
 
+Opt-in [hosted raw processing](docs/hosted-raw-sync.md) lets `pg serve` parse
+uploaded sources directly into PostgreSQL. It requires explicit tenant/schema
+provisioning, a restricted runtime role, authentication and Linux isolation.
+Ordinary PG serving remains read-only; `pg push` refuses hosted-owned schemas.
+
 Single-target configs still use the legacy `[pg]` block. To manage more than one
 PostgreSQL destination, define named `[pg.NAME]` blocks and set `default_pg`
 when more than one target exists:
@@ -576,7 +585,9 @@ exclude_projects = ["scratch"]
 
 Named target names are normalized case-insensitively. `all`, `local`, and the
 legacy `[pg]` field names `url`, `schema`, `machine_name`, `allow_insecure`,
-`projects`, and `exclude_projects` cannot be used for `[pg.NAME]`.
+`projects`, `exclude_projects`, `raw_tenant`, `raw_derivation`,
+`raw_poll_seconds`, `raw_attempt_seconds`, and `raw_max_attempts` cannot be used
+for `[pg.NAME]`.
 
 `AGENTSVIEW_PG_URL`, `AGENTSVIEW_PG_SCHEMA`, and `AGENTSVIEW_PG_MACHINE` still
 work, but in named-target mode they apply only to the effective default target.

cmd/agentsview/cli.go

@@ -651,6 +651,8 @@ func newPGCommand() *cobra.Command {
 	cmd.AddCommand(newPGServeCommand())
 	cmd.AddCommand(newPGVectorsCommand())
 	cmd.AddCommand(newPGServiceCommand())
+	cmd.AddCommand(newPGHostedProvisionCommand())
+	cmd.AddCommand(newPGRawReparseCommand())
 	return cmd
 }
 

cmd/agentsview/main.go

@@ -24,6 +24,7 @@ import (
 	"go.kenn.io/agentsview/internal/config"
 	"go.kenn.io/agentsview/internal/db"
 	"go.kenn.io/agentsview/internal/parser"
+	"go.kenn.io/agentsview/internal/rawderive"
 	"go.kenn.io/agentsview/internal/recall/extract"
 	"go.kenn.io/agentsview/internal/remotesync"
 	"go.kenn.io/agentsview/internal/secrets"
@@ -60,6 +61,9 @@ const (
 )
 
 func main() {
+	if handled, code := rawderive.RunParserChild(os.Args[1:]); handled {
+		os.Exit(code)
+	}
 	// Turn on the agentsview-test-fixture deny-list before any scan
 	// runs. The secrets package keeps the filter off by default so unit
 	// tests in this repo (which use the same random-looking fixtures

cmd/agentsview/pg.go

@@ -114,6 +114,11 @@ func runPGPush(
 		return err
 	}
 
+	for _, target := range targets {
+		if target.PG.RawTenant != "" || target.PG.RawDerivation {
+			return errors.New("pg push cannot mutate a hosted-owned projection")
+		}
+	}
 	applyClassifierConfig(appCfg)
 	ctx, stop := signal.NotifyContext(
 		context.Background(), os.Interrupt,
@@ -172,6 +177,9 @@ func runPGPushTarget(
 	if err != nil {
 		return err
 	}
+	if target.PG.RawTenant != "" || target.PG.RawDerivation {
+		return errors.New("pg push cannot mutate a hosted-owned projection")
+	}
 	if target.PG.URL == "" {
 		return fmt.Errorf("url not configured")
 	}
@@ -479,11 +487,12 @@ func loadPGServeConfig(cmd *cobra.Command) (config.Config, string, error) {
 }
 
 type pgServeStartup struct {
-	cfg     config.Config
-	ctx     context.Context
-	rtOpts  serveRuntimeOptions
-	srv     *server.Server
-	cleanup func()
+	cfg         config.Config
+	ctx         context.Context
+	rtOpts      serveRuntimeOptions
+	srv         *server.Server
+	cleanup     func()
+	startWorker func()
 }
 
 var preparePGServe = preparePGServeImpl
@@ -501,6 +510,12 @@ func preparePGServeImpl(appCfg config.Config, basePath string) (pgServeStartup,
 		return pgServeStartup{}, errors.New("pg serve: url not configured")
 	}
 
+	if err := pgCfg.ValidateRawDerivation(appCfg.RequireAuth); err != nil {
+		return pgServeStartup{}, err
+	}
+	if pgCfg.RawTenant != "" {
+		return prepareHostedPGServe(appCfg, pgCfg, basePath)
+	}
 	applyClassifierConfig(appCfg)
 	store, err := postgres.NewStore(
 		pgCfg.URL, pgCfg.Schema, pgCfg.AllowInsecure,
@@ -632,8 +647,16 @@ func runPGServe(appCfg config.Config, basePath string) {
 	if err != nil {
 		fatal("%v", err)
 	}
+	if err = runPreparedPGServe(startup); err != nil {
+		fatal("%v", err)
+	}
+}
+
+// Return through cleanup before the outer CLI may call os.Exit, including
+// readiness failures and unexpected server/proxy exits.
+func runPreparedPGServe(startup pgServeStartup) error {
 	defer startup.cleanup()
-	appCfg = startup.cfg
+	appCfg := startup.cfg
 	ctx := startup.ctx
 	rtOpts := startup.rtOpts
 	srv := startup.srv
@@ -646,11 +669,14 @@ func runPGServe(appCfg config.Config, basePath string) {
 	)
 	if err != nil {
 		if errors.Is(err, context.Canceled) {
-			return
+			return nil
 		}
-		fatal("pg serve: %v", err)
+		return fmt.Errorf("pg serve: %w", err)
 	}
 
+	if startup.startWorker != nil {
+		startup.startWorker()
+	}
 	// Write the kit runtime record so CLI commands can discover this
 	// daemon. ReadOnly=true marks it as pg serve (read-only)
 	// so clients can select an appropriate transport.
@@ -677,8 +703,9 @@ func runPGServe(appCfg config.Config, basePath string) {
 	}
 
 	if err := waitForServerRuntime(ctx, srv, rt); err != nil {
-		fatal("pg serve: %v", err)
+		return fmt.Errorf("pg serve: %w", err)
 	}
+	return nil
 }
 
 func writePGServeRuntimeRecord(rt *serveRuntime) bool {