diff --git a/server/cmd/multica/cmd_hook.go b/server/cmd/multica/cmd_hook.go new file mode 100644 index 00000000000..a315c3961ac --- /dev/null +++ b/server/cmd/multica/cmd_hook.go @@ -0,0 +1,346 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/multica-ai/multica/server/internal/cli" +) + +var hookCmd = &cobra.Command{ + Use: "hook", + Short: "Work with event hooks (automation rules)", +} + +var hookListCmd = &cobra.Command{ + Use: "list", + Short: "List hooks in the workspace", + RunE: runHookList, +} + +var hookGetCmd = &cobra.Command{ + Use: "get ", + Short: "Get a hook and its active revision", + Args: exactArgs(1), + RunE: runHookGet, +} + +var hookCreateCmd = &cobra.Command{ + Use: "create", + Short: "Create a hook from a JSON spec file", + RunE: runHookCreate, +} + +var hookUpdateCmd = &cobra.Command{ + Use: "update ", + Short: "Update a hook from a JSON spec file (appends a new revision)", + Args: exactArgs(1), + RunE: runHookUpdate, +} + +var hookEnableCmd = &cobra.Command{ + Use: "enable ", + Short: "Enable a hook", + Args: exactArgs(1), + RunE: runHookEnable, +} + +var hookDisableCmd = &cobra.Command{ + Use: "disable ", + Short: "Disable a hook", + Args: exactArgs(1), + RunE: runHookDisable, +} + +var hookDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Archive (soft-delete) a hook", + Args: exactArgs(1), + RunE: runHookDelete, +} + +var hookExecutionsCmd = &cobra.Command{ + Use: "executions ", + Short: "List a hook's recent execution trace", + Args: exactArgs(1), + RunE: runHookExecutions, +} + +var hookDryRunCmd = &cobra.Command{ + Use: "dry-run", + Short: "Evaluate a candidate hook spec against a historical event (read-only)", + RunE: runHookDryRun, +} + +var hookExplainCmd = &cobra.Command{ + Use: "explain ", + Short: "Explain whether a stored hook would fire for a historical event (read-only)", + Args: exactArgs(1), + RunE: runHookExplain, +} + +func init() { + hookCmd.AddCommand(hookListCmd) + hookCmd.AddCommand(hookGetCmd) + hookCmd.AddCommand(hookCreateCmd) + hookCmd.AddCommand(hookUpdateCmd) + hookCmd.AddCommand(hookEnableCmd) + hookCmd.AddCommand(hookDisableCmd) + hookCmd.AddCommand(hookDeleteCmd) + hookCmd.AddCommand(hookExecutionsCmd) + hookCmd.AddCommand(hookDryRunCmd) + hookCmd.AddCommand(hookExplainCmd) + + hookListCmd.Flags().String("output", "table", "Output format: table or json") + hookGetCmd.Flags().String("output", "json", "Output format: table or json") + hookExecutionsCmd.Flags().String("output", "table", "Output format: table or json") + + hookCreateCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookCreateCmd.MarkFlagRequired("file") + hookUpdateCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookUpdateCmd.MarkFlagRequired("file") + + hookDisableCmd.Flags().String("reason", "", "Optional reason recorded on the hook") + + hookDryRunCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookDryRunCmd.MarkFlagRequired("file") + hookDryRunCmd.Flags().String("event", "", "Historical event id to evaluate against (required)") + _ = hookDryRunCmd.MarkFlagRequired("event") + + hookExplainCmd.Flags().String("event", "", "Historical event id to evaluate against (required)") + _ = hookExplainCmd.MarkFlagRequired("event") + hookExplainCmd.Flags().Int("revision", 0, "Explain a specific revision number (default: active revision)") +} + +// readHookSpecFile loads and validates that the spec file is well-formed JSON, +// then returns it for the request body. The server performs full typed validation. +func readHookSpecFile(path string) (any, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read spec file: %w", err) + } + var body any + if err := json.Unmarshal(data, &body); err != nil { + return nil, fmt.Errorf("spec file is not valid JSON: %w", err) + } + return body, nil +} + +func runHookList(cmd *cobra.Command, _ []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var hooks []map[string]any + if err := client.GetJSON(ctx, "/api/hooks", &hooks); err != nil { + return fmt.Errorf("list hooks: %w", err) + } + output, _ := cmd.Flags().GetString("output") + if output == "json" { + return cli.PrintJSON(os.Stdout, hooks) + } + headers := []string{"ID", "NAME", "EVENT", "FIRE", "ENABLED"} + rows := make([][]string, 0, len(hooks)) + for _, hook := range hooks { + rows = append(rows, []string{ + strVal(hook, "id"), + strVal(hook, "name"), + hookRevisionField(hook, "event"), + hookRevisionField(hook, "fire_mode"), + fmt.Sprintf("%v", hook["enabled"]), + }) + } + cli.PrintTable(os.Stdout, headers, rows) + return nil +} + +func runHookGet(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var hook map[string]any + if err := client.GetJSON(ctx, "/api/hooks/"+args[0], &hook); err != nil { + return fmt.Errorf("get hook: %w", err) + } + return cli.PrintJSON(os.Stdout, hook) +} + +func runHookCreate(cmd *cobra.Command, _ []string) error { + path, _ := cmd.Flags().GetString("file") + body, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks", body, &result); err != nil { + return fmt.Errorf("create hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookUpdate(cmd *cobra.Command, args []string) error { + path, _ := cmd.Flags().GetString("file") + body, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PatchJSON(ctx, "/api/hooks/"+args[0], body, &result); err != nil { + return fmt.Errorf("update hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookEnable(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/"+args[0]+"/enable", map[string]any{}, &result); err != nil { + return fmt.Errorf("enable hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookDisable(cmd *cobra.Command, args []string) error { + reason, _ := cmd.Flags().GetString("reason") + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/"+args[0]+"/disable", map[string]any{"reason": reason}, &result); err != nil { + return fmt.Errorf("disable hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookDelete(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + if err := client.DeleteJSON(ctx, "/api/hooks/"+args[0]); err != nil { + return fmt.Errorf("delete hook: %w", err) + } + fmt.Printf("Hook archived: %s\n", args[0]) + return nil +} + +func runHookExecutions(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var execs []map[string]any + if err := client.GetJSON(ctx, "/api/hooks/"+args[0]+"/executions", &execs); err != nil { + return fmt.Errorf("list hook executions: %w", err) + } + output, _ := cmd.Flags().GetString("output") + if output == "json" { + return cli.PrintJSON(os.Stdout, execs) + } + headers := []string{"ID", "STATUS", "SKIP_REASON", "EVENT_ID", "CREATED_AT"} + rows := make([][]string, 0, len(execs)) + for _, e := range execs { + rows = append(rows, []string{ + strVal(e, "id"), + strVal(e, "status"), + strVal(e, "skip_reason"), + strVal(e, "event_id"), + strVal(e, "created_at"), + }) + } + cli.PrintTable(os.Stdout, headers, rows) + return nil +} + +func runHookDryRun(cmd *cobra.Command, _ []string) error { + path, _ := cmd.Flags().GetString("file") + event, _ := cmd.Flags().GetString("event") + spec, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/dry-run", map[string]any{"hook": spec, "event_id": event}, &result); err != nil { + return fmt.Errorf("dry-run hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookExplain(cmd *cobra.Command, args []string) error { + event, _ := cmd.Flags().GetString("event") + revision, _ := cmd.Flags().GetInt("revision") + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + body := map[string]any{"hook_id": args[0], "event_id": event} + if revision > 0 { + body["revision"] = revision + } + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/explain", body, &result); err != nil { + return fmt.Errorf("explain hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +// hookRevisionField reads a field out of the nested "revision" object for the +// list table view. +func hookRevisionField(hook map[string]any, key string) string { + rev, ok := hook["revision"].(map[string]any) + if !ok { + return "" + } + return strVal(rev, key) +} diff --git a/server/cmd/multica/main.go b/server/cmd/multica/main.go index 7491fd18da7..9d521b5b941 100644 --- a/server/cmd/multica/main.go +++ b/server/cmd/multica/main.go @@ -53,6 +53,7 @@ func init() { repoCmd.GroupID = groupCore skillCmd.GroupID = groupCore squadCmd.GroupID = groupCore + hookCmd.GroupID = groupCore chatCmd.GroupID = groupCore // Runtime commands @@ -79,6 +80,7 @@ func init() { rootCmd.AddCommand(repoCmd) rootCmd.AddCommand(skillCmd) rootCmd.AddCommand(squadCmd) + rootCmd.AddCommand(hookCmd) rootCmd.AddCommand(chatCmd) rootCmd.AddCommand(daemonCmd) rootCmd.AddCommand(runtimeCmd) diff --git a/server/cmd/server/domain_event_retention.go b/server/cmd/server/domain_event_retention.go new file mode 100644 index 00000000000..40f6e73baa1 --- /dev/null +++ b/server/cmd/server/domain_event_retention.go @@ -0,0 +1,24 @@ +package main + +import ( + "context" + "log/slog" +) + +// Domain event retention (MUL-4332 §4.1 / §9) is DELIBERATELY an explicit no-op +// in PR1. The correct retention predicate is "dispatched AND older than the TTL +// AND every related hook_execution is terminal" — but hook_execution does not +// exist until PR3, and PR1 has no dispatcher, so nothing is ever eligible for +// deletion. Shipping a weaker "dispatched + TTL" sweep now would, the moment PR3 +// flips dispatching on, risk reclaiming events whose executions are still +// running or retrying (review point 5). Retention therefore lands in PR3 +// alongside hook_execution, the full terminal predicate, and concurrent-sweeper +// tests. +// +// The worker is kept wired (rather than silently omitted) so the intent is +// visible at the call site: it logs once and then idles until shutdown, doing no +// deletes. +func runDomainEventRetention(ctx context.Context) { + slog.Info("domain event retention: explicit no-op in PR1, deferred to PR3 (needs hook_execution terminal predicate)") + <-ctx.Done() +} diff --git a/server/cmd/server/hook_executor.go b/server/cmd/server/hook_executor.go new file mode 100644 index 00000000000..426b409542d --- /dev/null +++ b/server/cmd/server/hook_executor.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// hookExecutorTick is how often the executor polls for runnable executions. It +// stays dormant (a cheap flag check) while automation_event_hooks is off. +const hookExecutorTick = 2 * time.Second + +// runHookExecutor is the Event Hooks executor loop (MUL-4332 PR3 §7.2). It is gated +// on its OWN default-off switch, separate from the one that opens the policy API, +// dry-run and the matcher: enabling the engine for shadow evaluation must never +// start performing real side effects. Only automation_event_hook_execution (on top +// of automation_event_hooks) lets this loop claim queued executions. +func runHookExecutor(ctx context.Context, svc *service.HookService, flags *featureflag.Service) { + if svc == nil { + return + } + ticker := time.NewTicker(hookExecutorTick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !featureflags.EventHookExecutionEnabled(ctx, flags) { + continue + } + if _, err := svc.ClaimAndRun(ctx, service.ExecutorBatchSize); err != nil { + slog.Warn("hook executor tick failed", "error", err) + } + } + } +} diff --git a/server/cmd/server/hook_matcher.go b/server/cmd/server/hook_matcher.go new file mode 100644 index 00000000000..a9b0cc5dc01 --- /dev/null +++ b/server/cmd/server/hook_matcher.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// hookMatcherTick is how often the matcher polls the outbox. It stays dormant +// (a cheap flag check) while automation_event_hooks is off. +const hookMatcherTick = 2 * time.Second + +// runHookMatcher is the durable Event Hooks matcher loop (MUL-4332 PR3). It only +// claims and matches pending domain events when the automation_event_hooks flag +// is enabled, so with the default-off flag it does nothing and production +// behaviour is unchanged. The matcher only records queued/skipped decisions; it +// runs no actions (the executor is a later slice). +func runHookMatcher(ctx context.Context, svc *service.HookService, flags *featureflag.Service) { + if svc == nil { + return + } + ticker := time.NewTicker(hookMatcherTick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !featureflags.EventHooksEnabled(ctx, flags) { + continue + } + if _, err := svc.ClaimAndMatch(ctx, service.MatcherBatchSize); err != nil { + slog.Warn("hook matcher tick failed", "error", err) + } + } + } +} diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index 1d753f6b191..94a50072a39 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -386,6 +386,14 @@ func main() { // Start background sweeper to mark stale runtimes as offline. go runRuntimeSweeper(sweepCtx, queries, liveness, taskSvc, bus) + // Domain event retention (MUL-4332) — explicit no-op in PR1, wired for + // visibility; the real sweep lands in PR3 with hook_execution. + go runDomainEventRetention(sweepCtx) + // Event Hooks matcher (MUL-4332 PR3) — consumes the domain_event outbox and + // records queued/skipped hook_execution decisions. Gated on the + // automation_event_hooks flag (default off), so it is dormant until enabled. + go runHookMatcher(sweepCtx, h.HookService, flags) + go runHookExecutor(sweepCtx, h.HookService, flags) go heartbeatScheduler.Run(sweepCtx) go runAutopilotFailureMonitor(autopilotCtx, queries, bus, envFailureMonitorConfig()) go runDBStatsLogger(sweepCtx, pool) diff --git a/server/cmd/server/router.go b/server/cmd/server/router.go index 9b1eb3e7d50..e5127b4c5c3 100644 --- a/server/cmd/server/router.go +++ b/server/cmd/server/router.go @@ -1248,6 +1248,31 @@ func NewRouterWithOptions(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus }) }) + // Event Hooks (MUL-4332) - automation rules CRUD. Every handler + // additionally gates on the automation_event_hooks feature flag + // (default off), so this surface is invisible until enabled. + r.Route("/api/hooks", func(r chi.Router) { + r.Get("/", h.ListHooks) + r.Post("/", h.CreateHook) + // Read-only debug surface (PR3, decision 2A): evaluate a candidate + // spec or a stored hook against a historical event without executing + // anything. chi matches these static routes before the /{id} param. + r.Post("/dry-run", h.DryRunHook) + r.Post("/explain", h.ExplainHook) + r.Route("/{id}", func(r chi.Router) { + r.Get("/", h.GetHook) + r.Patch("/", h.UpdateHook) + r.Delete("/", h.DeleteHook) + r.Post("/enable", h.EnableHook) + r.Post("/disable", h.DisableHook) + r.Get("/executions", h.ListHookExecutions) + }) + }) + + // Event Hooks correlation-chain read (PR3, decision 2A), gated on the + // same automation_event_hooks flag inside the handler. + r.Get("/api/events", h.ListEventsByCorrelation) + // Dashboard — workspace-wide token + run-time rollups for the // "/{slug}/dashboard" page. Optional ?project_id filter scopes // the rollup to a single project. diff --git a/server/cmd/server/runtime_sweeper.go b/server/cmd/server/runtime_sweeper.go index 941e01c13e3..a3a9cad143c 100644 --- a/server/cmd/server/runtime_sweeper.go +++ b/server/cmd/server/runtime_sweeper.go @@ -76,6 +76,12 @@ const ( chatFinalizeGraceSeconds = 60.0 // chatFinalizeBatchSize caps deferred finalizations per tick. chatFinalizeBatchSize = 100 + // bulkFailBatchSize caps how many tasks a single offline-runtime or + // stale-task sweep fails per tick (MUL-4332 review point 2). Bounding the + // candidate SELECT keeps the FOR UPDATE lock hold short even if a large + // backlog of orphans accumulates; the rest drain on later ticks. 500 mirrors + // queuedExpireBatchSize — far above any realistic per-tick orphan count. + bulkFailBatchSize = 500 ) // runRuntimeSweeper periodically marks runtimes as offline if their @@ -99,6 +105,7 @@ func runRuntimeSweeper(ctx context.Context, queries *db.Queries, liveness handle return case <-ticker.C: sweepStaleRuntimes(ctx, queries, liveness, taskSvc, bus) + sweepOfflineRuntimeTasks(ctx, taskSvc) sweepStaleTasks(ctx, queries, taskSvc, bus) sweepExpiredQueuedTasks(ctx, queries, taskSvc) sweepDeferredChatFinalizations(ctx, queries, taskSvc) @@ -167,14 +174,11 @@ func sweepStaleRuntimes(ctx context.Context, queries *db.Queries, liveness handl slog.Info("runtime sweeper: marked stale runtimes offline", "count", len(staleRows), "workspaces", len(workspaces)) - // Fail orphaned tasks (dispatched/running) whose runtimes just went offline. - failedTasks, err := queries.FailTasksForOfflineRuntimes(ctx) - if err != nil { - slog.Warn("runtime sweeper: failed to clean up stale tasks", "error", err) - } else if len(failedTasks) > 0 { - slog.Info("runtime sweeper: failed orphaned tasks", "count", len(failedTasks)) - taskSvc.HandleFailedTasks(ctx, failedTasks) - } + // Orphaned tasks whose runtimes just went offline are reclaimed by + // sweepOfflineRuntimeTasks, which runs every tick against ALL offline + // runtimes (MUL-4332 review point 2) — not only the ones flipped this tick — + // so a rolled-back batch or a runtime that went offline via another path is + // still retried on the next tick rather than stranded here. // Notify frontend clients so they re-fetch runtime list. for wsID := range workspaces { @@ -269,13 +273,26 @@ func gcRuntimes(ctx context.Context, queries *db.Queries, bus *events.Bus) { // edge where a runtime row lingers online-with-stale-heartbeat past the // wall clock (MUL-4107). func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService, bus *events.Bus) { - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ - DispatchTimeoutSecs: dispatchTimeoutSeconds, - RunningTimeoutSecs: runningTimeoutSeconds, - // Reuse the runtime stale window so the running-task backstop - // exactly matches what sweepStaleRuntimes considers "not alive". - RuntimeStaleSecs: staleThresholdSeconds, - }) + // Select a bounded batch of stale candidates, then fail the resolvable ones + // with their task.failed events atomically (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectStaleTasksToFail(ctx, db.SelectStaleTasksToFailParams{ + DispatchTimeoutSecs: dispatchTimeoutSeconds, + RunningTimeoutSecs: runningTimeoutSeconds, + // Reuse the runtime stale window so the running-task backstop + // exactly matches what sweepStaleRuntimes considers "not alive". + RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, + }) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "task timed out", Valid: true}, + FailureReason: pgtype.Text{String: "timeout", Valid: true}, + }) + }) if err != nil { slog.Warn("task sweeper: failed to clean up stale tasks", "error", err) return @@ -289,6 +306,40 @@ func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service. taskSvc.HandleFailedTasks(ctx, failedTasks) } +// sweepOfflineRuntimeTasks fails orphaned dispatched/running/waiting tasks whose +// runtime is offline, then runs the standard post-fail side effects. Unlike the +// old placement inside sweepStaleRuntimes (which only fired when a runtime was +// flipped offline THIS tick), it runs EVERY tick against ALL offline runtimes +// (MUL-4332 review point 2): a rolled-back batch, a transient event failure, or a +// runtime taken offline via another path is retried on the next tick instead of +// stranding its `waiting_local_directory` orphans. Poison rows are isolated +// inside FailBulkTasksWithEvents so one corrupt row never blocks the rest. +func sweepOfflineRuntimeTasks(ctx context.Context, taskSvc *service.TaskService) { + if taskSvc == nil { + return + } + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectTasksForOfflineRuntimes(ctx, bulkFailBatchSize) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + slog.Warn("runtime sweeper: failed to clean up offline-runtime tasks", "error", err) + return + } + if len(failedTasks) == 0 { + return + } + slog.Info("runtime sweeper: failed orphaned tasks", "count", len(failedTasks)) + taskSvc.HandleFailedTasks(ctx, failedTasks) +} + // sweepExpiredQueuedTasks fails tasks that have been sitting in 'queued' for // longer than the TTL. Companion to the dispatch-time admission gate added // in MUL-1899: that gate prevents new doomed enqueues; this gate drains the @@ -296,10 +347,22 @@ func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service. // a task is already queued. Capped to queuedExpireBatchSize per tick so a // big backlog can't monopolise the DB. func sweepExpiredQueuedTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService) { - failedTasks, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ - TtlSecs: queuedTTLSeconds, - MaxPerTick: queuedExpireBatchSize, - }) + // Select a bounded batch of TTL-expired queued candidates, then fail the + // resolvable ones with their events atomically (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectExpiredQueuedTasks(ctx, db.SelectExpiredQueuedTasksParams{ + TtlSecs: queuedTTLSeconds, + MaxPerTick: queuedExpireBatchSize, + }) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "task expired in queue", Valid: true}, + FailureReason: pgtype.Text{String: "queued_expired", Valid: true}, + }) + }) if err != nil { slog.Warn("task sweeper: failed to expire stale queued tasks", "error", err) return diff --git a/server/cmd/server/runtime_sweeper_test.go b/server/cmd/server/runtime_sweeper_test.go index 050bd33e967..402268359ff 100644 --- a/server/cmd/server/runtime_sweeper_test.go +++ b/server/cmd/server/runtime_sweeper_test.go @@ -7,10 +7,55 @@ import ( "testing" "time" + "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/events" db "github.com/multica-ai/multica/server/pkg/db/generated" ) +// selectAndFailStale / selectAndFailExpiredQueued run the two-step MUL-4332 +// bulk-fail (candidate SELECT then FailAgentTasksByIDs) that replaced the old +// single-statement UPDATE queries, so these predicate tests keep asserting on +// genuinely-failed rows. Poison isolation and workspace resolution are covered +// separately by the FailBulkTasksWithEvents service tests; here we exercise the +// raw SQL predicate + fail. +func selectAndFailStale(t *testing.T, ctx context.Context, queries *db.Queries, p db.SelectStaleTasksToFailParams) []db.AgentTaskQueue { + t.Helper() + candidates, err := queries.SelectStaleTasksToFail(ctx, p) + if err != nil { + t.Fatalf("SelectStaleTasksToFail failed: %v", err) + } + return failCandidatesByIDs(t, ctx, queries, candidates, "task timed out", "timeout") +} + +func selectAndFailExpiredQueued(t *testing.T, ctx context.Context, queries *db.Queries, p db.SelectExpiredQueuedTasksParams) []db.AgentTaskQueue { + t.Helper() + candidates, err := queries.SelectExpiredQueuedTasks(ctx, p) + if err != nil { + t.Fatalf("SelectExpiredQueuedTasks failed: %v", err) + } + return failCandidatesByIDs(t, ctx, queries, candidates, "task expired in queue", "queued_expired") +} + +func failCandidatesByIDs(t *testing.T, ctx context.Context, queries *db.Queries, candidates []db.AgentTaskQueue, errMsg, reason string) []db.AgentTaskQueue { + t.Helper() + if len(candidates) == 0 { + return nil + } + ids := make([]pgtype.UUID, len(candidates)) + for i, c := range candidates { + ids[i] = c.ID + } + failed, err := queries.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: errMsg, Valid: true}, + FailureReason: pgtype.Text{String: reason, Valid: true}, + }) + if err != nil { + t.Fatalf("FailAgentTasksByIDs failed: %v", err) + } + return failed +} + // setupSweeperTestFixture creates an issue and a task in the given status with // timestamps old enough to trigger the sweeper. Returns (issueID, agentID, taskID). func setupSweeperTestFixture(t *testing.T, taskStatus string) (string, string, string) { @@ -175,14 +220,12 @@ func TestSweepStaleTasksBroadcastsWithWorkspaceID(t *testing.T) { }) // Use very short timeouts to trigger the sweep on our test task - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, // 1 second — our task is 3 hours old RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks query failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale task to be failed") } @@ -225,7 +268,7 @@ func TestSweepStaleTasksBroadcastsWithWorkspaceID(t *testing.T) { // Verify DB: task should be failed var status string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) if err != nil { t.Fatalf("failed to query task status: %v", err) } @@ -259,14 +302,12 @@ func TestSweepStaleTasksReconcileAgentStatus(t *testing.T) { }) // Fail stale tasks with short timeout - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale task") } @@ -275,7 +316,7 @@ func TestSweepStaleTasksReconcileAgentStatus(t *testing.T) { // Verify agent status is now "idle" in DB var agentStatus string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent WHERE id = $1`, agentID).Scan(&agentStatus) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent WHERE id = $1`, agentID).Scan(&agentStatus) if err != nil { t.Fatalf("failed to query agent status: %v", err) } @@ -321,16 +362,14 @@ func TestSweepDispatchedStaleTask(t *testing.T) { }) // Fail stale tasks — dispatch timeout of 1 second (our task is 10 minutes old) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 1.0, RunningTimeoutSecs: 9000.0, // RuntimeStaleSecs only affects the running branch — irrelevant for // this dispatched-timeout test, but wired for API consistency. RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale dispatched task") } @@ -339,7 +378,7 @@ func TestSweepDispatchedStaleTask(t *testing.T) { // Verify DB: task should be failed var status string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) if err != nil { t.Fatalf("failed to query task: %v", err) } @@ -397,14 +436,12 @@ func TestSweepRunningTaskSkippedWhenRuntimeFresh(t *testing.T) { // Task started_at is 3h ago; RunningTimeoutSecs=1s would kill on wall clock // alone — but the runtime is proving liveness, so the sweeper must skip it. queries := db.New(testPool) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } for _, ft := range failedTasks { if ft.ID.Bytes == parseUUIDBytes(taskID) { @@ -440,14 +477,12 @@ func TestSweepRunningTaskKilledWhenRuntimeStale(t *testing.T) { ageOutAgentRuntime(t, agentID, 10*time.Minute) queries := db.New(testPool) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } found := false for _, ft := range failedTasks { @@ -531,14 +566,12 @@ func TestSweepResetsInProgressIssueToTodo(t *testing.T) { ageOutAgentRuntime(t, agentID, 10*time.Minute) // Fail the stale task (running timeout of 1 second — our task is 3 hours old). - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, ctx, queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } // Confirm our task was swept. found := false @@ -619,14 +652,12 @@ func TestSweepDoesNotResetIssueAlreadyInReview(t *testing.T) { // Runtime must be stale for the running-task wall clock to fire (MUL-4107). ageOutAgentRuntime(t, agentID, 10*time.Minute) - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, ctx, queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } broadcastFailedTasks(ctx, queries, nil, bus, failedTasks) @@ -706,13 +737,10 @@ func TestExpireStaleQueuedTasks(t *testing.T) { } queries := db.New(testPool) - failed, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ + failed := selectAndFailExpiredQueued(t, ctx, queries, db.SelectExpiredQueuedTasksParams{ TtlSecs: 3600.0, // 1h TTL — old task is 5h, fresh task is 0s MaxPerTick: 100, }) - if err != nil { - t.Fatalf("ExpireStaleQueuedTasks failed: %v", err) - } if len(failed) != 1 { t.Fatalf("expected exactly 1 expired task, got %d", len(failed)) } @@ -802,13 +830,10 @@ func TestExpireStaleQueuedTasksRespectsBatchLimit(t *testing.T) { } queries := db.New(testPool) - failed, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ + failed := selectAndFailExpiredQueued(t, ctx, queries, db.SelectExpiredQueuedTasksParams{ TtlSecs: 3600.0, MaxPerTick: 2, // cap below the backlog }) - if err != nil { - t.Fatalf("ExpireStaleQueuedTasks failed: %v", err) - } if len(failed) != 2 { t.Fatalf("expected batch cap of 2, got %d", len(failed)) } diff --git a/server/internal/admission/admission.go b/server/internal/admission/admission.go new file mode 100644 index 00000000000..1e26faafca1 --- /dev/null +++ b/server/internal/admission/admission.go @@ -0,0 +1,83 @@ +// Package admission holds the pure member-permission predicates shared by the +// HTTP handlers and the automation hook service, so both judge agent invocation +// and autopilot write access with identical semantics. The functions here take +// already-loaded rows and make no DB calls, which lets the hook service evaluate +// them inside its write transaction against a hook's stored principal (MUL-4332 +// PR2 review round 3). +package admission + +import ( + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// MemberHitsInvocationTargets reports whether a member is on a public_to agent's +// allow-list: a workspace target admits any member; a member target admits the +// matching user; team targets are inert in v1. +func MemberHitsInvocationTargets(targets []db.AgentInvocationTarget, userID string) bool { + for _, t := range targets { + switch t.TargetType { + case "workspace": + return true + case "member": + if util.UUIDToString(t.TargetID) == userID { + return true + } + } + } + return false +} + +// AgentInvocableByMember decides whether a member principal may invoke an agent, +// for the hook service's fail-closed save-time admission. A principal who is not +// a CURRENT member of the workspace can invoke nothing — not even an agent they +// own — because a hook must run under a live, accountable member (review round 4, +// point 1); this is stricter than the interactive Handler.canInvokeAgent, whose +// owner branch is membership-independent. Given a current member: the agent owner +// may invoke; otherwise a public_to agent is invocable only when the member is on +// its allow-list. There is no admin bypass — an admin editing a hook may not +// grant a stored principal reach the principal does not have. +func AgentInvocableByMember(agent db.Agent, targets []db.AgentInvocationTarget, memberUserID string, isCurrentMember bool) bool { + if !isCurrentMember { + return false + } + if memberUserID != "" && util.UUIDToString(agent.OwnerID) == memberUserID { + return true + } + if agent.PermissionMode != "public_to" { + return false + } + for _, t := range targets { + switch t.TargetType { + case "workspace": + if isCurrentMember { + return true + } + case "member": + if memberUserID != "" && util.UUIDToString(t.TargetID) == memberUserID { + return true + } + } + } + return false +} + +// AutopilotWriteByOwnership reports whether a member may write an autopilot by +// role or authorship (workspace owner/admin, or the member who created it). +// Collaborator grants are checked separately by the caller (a DB lookup). +func AutopilotWriteByOwnership(ap db.Autopilot, member db.Member) bool { + if RoleAllowed(member.Role, "owner", "admin") { + return true + } + return ap.CreatedByType == "member" && util.UUIDToString(ap.CreatedByID) == util.UUIDToString(member.UserID) +} + +// RoleAllowed reports whether role is one of the allowed workspace roles. +func RoleAllowed(role string, allowed ...string) bool { + for _, a := range allowed { + if role == a { + return true + } + } + return false +} diff --git a/server/internal/admission/admission_test.go b/server/internal/admission/admission_test.go new file mode 100644 index 00000000000..b9bb567c019 --- /dev/null +++ b/server/internal/admission/admission_test.go @@ -0,0 +1,67 @@ +package admission + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgtype" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +func uid(t *testing.T, s string) pgtype.UUID { + t.Helper() + var u pgtype.UUID + if err := u.Scan(s); err != nil { + t.Fatalf("bad uuid %q: %v", s, err) + } + return u +} + +const ( + owner = "11111111-1111-1111-1111-111111111111" + member = "22222222-2222-2222-2222-222222222222" +) + +func TestAgentInvocableByMember(t *testing.T) { + privateAgent := db.Agent{OwnerID: uid(t, owner), PermissionMode: "private"} + publicAgent := db.Agent{OwnerID: uid(t, owner), PermissionMode: "public_to"} + wsTarget := []db.AgentInvocationTarget{{TargetType: "workspace"}} + memberTarget := []db.AgentInvocationTarget{{TargetType: "member", TargetID: uid(t, member)}} + + if !AgentInvocableByMember(privateAgent, nil, owner, true) { + t.Error("owner must be able to invoke their own private agent") + } + if AgentInvocableByMember(privateAgent, nil, owner, false) { + t.Error("a non-member must invoke nothing, even an agent they own") + } + if AgentInvocableByMember(privateAgent, wsTarget, member, true) { + t.Error("a private agent must not be invocable by a non-owner member") + } + if !AgentInvocableByMember(publicAgent, wsTarget, member, true) { + t.Error("a public_to workspace-target agent is invocable by a current member") + } + if AgentInvocableByMember(publicAgent, wsTarget, member, false) { + t.Error("a workspace target must NOT admit a non-member") + } + if !AgentInvocableByMember(publicAgent, memberTarget, member, true) { + t.Error("a member target admits the matching member") + } + if AgentInvocableByMember(publicAgent, memberTarget, "33333333-3333-3333-3333-333333333333", true) { + t.Error("a member target must not admit a different member") + } +} + +func TestAutopilotWriteByOwnership(t *testing.T) { + byMember := func(role string) db.Member { return db.Member{Role: role, UserID: uid(t, member)} } + ownedByMember := db.Autopilot{CreatedByType: "member", CreatedByID: uid(t, member)} + ownedByOther := db.Autopilot{CreatedByType: "member", CreatedByID: uid(t, owner)} + + if !AutopilotWriteByOwnership(ownedByOther, byMember("admin")) { + t.Error("admin may write any autopilot") + } + if !AutopilotWriteByOwnership(ownedByMember, byMember("member")) { + t.Error("the creating member may write their own autopilot") + } + if AutopilotWriteByOwnership(ownedByOther, byMember("member")) { + t.Error("a plain member may not write another member's autopilot") + } +} diff --git a/server/internal/automation/eval.go b/server/internal/automation/eval.go new file mode 100644 index 00000000000..7574c2f653d --- /dev/null +++ b/server/internal/automation/eval.go @@ -0,0 +1,347 @@ +package automation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" +) + +// This file is the single read-only evaluator for Event Hooks (MUL-4332 PR3, +// ratified decision 2A). The matcher (a later PR3 slice) and the dry-run/explain +// debug surface share it, so an explanation can never drift from real execution. +// It computes only WHETHER an event matches a revision and whether the revision's +// conditions hold against current workspace state; it performs NO action and +// mutates NO durable state (no rising-edge latch, rate bucket, execution or +// effect). The `when` match reads the event's own (historical) payload; the `if` +// conditions read current state via StateReader. + +// Stable reason codes. explain/dry-run and (later) the matcher's skip_reason draw +// from this fixed vocabulary so callers can branch on a stable string. +const ( + ReasonMatched = "matched" // matches and conditions currently hold + ReasonEventTypeMismatch = "event_type_mismatch" // the event is not the hook's event type + ReasonNoMatch = "no_match" // a when-clause did not match + ReasonConditionFalse = "condition_false" // matched, but an if-condition is not satisfied +) + +// EvaluatedAgainstCurrentState labels the state basis of a condition evaluation +// (2A): conditions read current workspace state, not the event's point in time. +const EvaluatedAgainstCurrentState = "current_state" + +// ErrInvalidConfig marks a DETERMINISTIC failure to interpret a stored revision: +// the configuration itself is unusable, so every retry fails identically. It is +// deliberately distinguishable from a transient StateReader/database error, which +// Evaluate propagates unwrapped. The matcher branches on this: a config error is +// isolated as one terminal `failed` execution so the remaining candidates for the +// same event still run, whereas a transient error retries the whole event. +var ErrInvalidConfig = errors.New("invalid hook configuration") + +// StateReader reads the current value of an issue field for condition evaluation. +// Implemented by the service against workspace-scoped queries; kept as an +// interface so this package stays pure and unit-testable. +type StateReader interface { + // IssueField returns the current value of field (status | assignee_id | + // parent_issue_id) for the issue. exists is false when the issue is absent + // from the workspace, in which case the predicate is treated as unsatisfied. + IssueField(ctx context.Context, issueID, field string) (value string, exists bool, err error) +} + +// EventView is the read-only projection of a domain event the evaluator matches +// against: envelope fields plus the decoded payload object. +type EventView struct { + Type string + SubjectID string + ActorType string + ActorID string + Payload map[string]any +} + +// EvalRevision is the parsed hook revision configuration the evaluator needs. +type EvalRevision struct { + EventType string + Match json.RawMessage + Conditions []ConditionSpec + FireMode string +} + +// ClauseResult is the per-field outcome of one when-match clause. It records the +// operator, the observed event value and presence, and the expected operand, so a +// single structured result feeds dry-run, explain and (next slice) the matcher's +// stored match_snapshot — no re-derivation, no parallel logic (review point 2). +type ClauseResult struct { + Field string `json:"field"` + Op string `json:"op"` + Observed string `json:"observed,omitempty"` + Present bool `json:"present"` + Expected []string `json:"expected,omitempty"` + Matched bool `json:"matched"` +} + +// IssueObserved records the observed current value of one issue field consulted +// by a condition, and whether that issue satisfied the predicate. +type IssueObserved struct { + ID string `json:"id"` + Field string `json:"field"` + Observed string `json:"observed,omitempty"` + Present bool `json:"present"` + Matched bool `json:"matched"` +} + +// ConditionResult is the structured outcome of one if-condition against current +// state: the operator/expected operand plus every observed input, so the same +// object is the condition_snapshot the matcher will store (review point 2). +type ConditionResult struct { + Kind string `json:"kind"` // issues_status | issue_field + Matched bool `json:"matched"` + Mode string `json:"mode,omitempty"` // all | any (issues_status) + Field string `json:"field,omitempty"` // issue field (issue_field) + Op string `json:"op,omitempty"` // eq | in (issue_field) + Expected []string `json:"expected,omitempty"` // target status / eq value / in-set + Issues []IssueObserved `json:"issues,omitempty"` // per-issue observed state +} + +// Evaluation is the complete read-only decision for one (event, revision) pair. +// +// Eligible means the when-match held and every if-condition is currently +// satisfied. It is NOT a fire decision: a rising-edge latch and the depth / rate +// / permission / fuse guards are not evaluated in read-only mode, so +// DecisionComplete is false and no field ever claims the rule "will execute" +// (review point 1). The matcher (next slice) will set DecisionComplete and the +// final fire verdict after evaluating the latch and guards. +type Evaluation struct { + Event string `json:"event"` + HookEvent string `json:"hook_event"` + FireMode string `json:"fire_mode"` + Matched bool `json:"matched"` + MatchClauses []ClauseResult `json:"match_clauses"` + ConditionsMet bool `json:"conditions_met"` + Conditions []ConditionResult `json:"conditions"` + Eligible bool `json:"eligible"` + DecisionComplete bool `json:"decision_complete"` + Reason string `json:"reason"` + EvaluatedAgainst string `json:"evaluated_against"` + Note string `json:"note,omitempty"` +} + +// MatchSnapshot / ConditionSnapshot serialize the structured inputs+conclusions +// exactly as the matcher will persist them, so dry-run/explain and the stored +// execution snapshot are byte-identical for the same (event, revision, state). +func (e Evaluation) MatchSnapshot() (json.RawMessage, error) { return json.Marshal(e.MatchClauses) } +func (e Evaluation) ConditionSnapshot() (json.RawMessage, error) { return json.Marshal(e.Conditions) } + +// Evaluate is the shared read-only decision. It never mutates durable state. +func Evaluate(ctx context.Context, event EventView, rev EvalRevision, state StateReader) (Evaluation, error) { + ev := Evaluation{ + Event: event.Type, + HookEvent: rev.EventType, + FireMode: rev.FireMode, + EvaluatedAgainst: EvaluatedAgainstCurrentState, + } + + // A revision only matches events of its own type. + if event.Type != rev.EventType { + ev.Reason = ReasonEventTypeMismatch + return ev, nil + } + + match, err := ParseMatch(rev.Match) + if err != nil { + return Evaluation{}, fmt.Errorf("%w: parse stored match: %v", ErrInvalidConfig, err) + } + ev.Matched, ev.MatchClauses = evalMatch(event, match) + if !ev.Matched { + ev.Reason = ReasonNoMatch + return ev, nil + } + + ev.ConditionsMet = true + for _, c := range rev.Conditions { + cr, err := evalCondition(ctx, c, state) + if err != nil { + return Evaluation{}, err + } + ev.Conditions = append(ev.Conditions, cr) + if !cr.Matched { + ev.ConditionsMet = false + } + } + if !ev.ConditionsMet { + ev.Reason = ReasonConditionFalse + return ev, nil + } + + // The event matches and conditions currently hold. This makes the rule + // ELIGIBLE, but not a completed fire decision: read-only evaluation does not + // consult the rising-edge latch or the depth/rate/permission/fuse guards, so + // DecisionComplete stays false and no field claims execution (review point 1). + ev.Eligible = true + ev.Reason = ReasonMatched + if rev.FireMode == FireRisingEdge { + ev.Note = "eligible now, but rising_edge fires only on a false→true edge; the durable latch and guards are not evaluated in read-only mode" + } else { + ev.Note = "eligible now; the depth/rate/permission/fuse guards are not evaluated in read-only mode" + } + return ev, nil +} + +// field resolves an event field path to a normalized string value. +func (e EventView) field(path string) (string, bool) { + switch path { + case "subject_id": + return e.SubjectID, e.SubjectID != "" + case "actor_type": + return e.ActorType, e.ActorType != "" + case "actor_id": + return e.ActorID, e.ActorID != "" + } + raw, ok := e.Payload[path] + if !ok { + return "", false + } + return normalizeScalar(raw) +} + +func evalMatch(event EventView, m Match) (bool, []ClauseResult) { + matched := true + results := make([]ClauseResult, 0, len(m)) + for field, clause := range m { + val, present := event.field(field) + ok := evalClause(clause, val, present) + if !ok { + matched = false + } + results = append(results, ClauseResult{ + Field: field, + Op: string(clause.Op), + Observed: val, + Present: present, + Expected: clauseExpected(clause), + Matched: ok, + }) + } + // Stable order for deterministic snapshots / responses. + sort.Slice(results, func(i, j int) bool { return results[i].Field < results[j].Field }) + return matched, results +} + +func evalClause(c MatchClause, val string, present bool) bool { + switch c.Op { + case MatchEq: + return present && val == c.Value + case MatchIn: + return present && contains(c.Set, val) + case MatchExists: + return present == c.Exists + } + return false +} + +func clauseExpected(c MatchClause) []string { + switch c.Op { + case MatchEq: + return []string{c.Value} + case MatchIn: + return c.Set + case MatchExists: + return []string{fmt.Sprintf("%t", c.Exists)} + } + return nil +} + +func evalCondition(ctx context.Context, c ConditionSpec, state StateReader) (ConditionResult, error) { + if c.IssuesStatus != nil { + return evalIssuesStatus(ctx, *c.IssuesStatus, state) + } + if c.IssueField != nil { + return evalIssueField(ctx, *c.IssueField, state) + } + // A stored, validated condition always has exactly one variant. + return ConditionResult{Kind: "unknown", Matched: false}, nil +} + +func evalIssuesStatus(ctx context.Context, c IssuesStatusCond, state StateReader) (ConditionResult, error) { + target, mode := c.All, "all" + if c.Any != "" { + target, mode = c.Any, "any" + } + allHit, anyHit := true, false + observed := make([]IssueObserved, 0, len(c.IDs)) + for _, id := range c.IDs { + status, exists, err := state.IssueField(ctx, id, IssueFieldStatus) + if err != nil { + return ConditionResult{}, err + } + hit := exists && status == target + if hit { + anyHit = true + } else { + allHit = false + } + observed = append(observed, IssueObserved{ID: id, Field: IssueFieldStatus, Observed: status, Present: exists, Matched: hit}) + } + met := allHit + if mode == "any" { + met = anyHit + } + return ConditionResult{ + Kind: "issues_status", + Matched: met, + Mode: mode, + Field: IssueFieldStatus, + Op: string(MatchEq), + Expected: []string{target}, + Issues: observed, + }, nil +} + +func evalIssueField(ctx context.Context, c IssueFieldCond, state StateReader) (ConditionResult, error) { + val, exists, err := state.IssueField(ctx, c.ID, c.Field) + if err != nil { + return ConditionResult{}, err + } + op, expected := string(MatchEq), []string{c.Eq} + met := false + if c.Eq == "" { + op, expected = string(MatchIn), c.In + } + if exists { + if c.Eq != "" { + met = val == c.Eq + } else { + met = contains(c.In, val) + } + } + return ConditionResult{ + Kind: "issue_field", + Matched: met, + Field: c.Field, + Op: op, + Expected: expected, + Issues: []IssueObserved{{ID: c.ID, Field: c.Field, Observed: val, Present: exists, Matched: met}}, + }, nil +} + +func normalizeScalar(v any) (string, bool) { + switch t := v.(type) { + case string: + return t, true + case bool: + return fmt.Sprintf("%t", t), true + case float64: + return formatNumber(t), true + default: + // null, arrays and objects are not matchable scalars. + return "", false + } +} + +func contains(set []string, v string) bool { + for _, s := range set { + if s == v { + return true + } + } + return false +} diff --git a/server/internal/automation/eval_test.go b/server/internal/automation/eval_test.go new file mode 100644 index 00000000000..61145ab31f0 --- /dev/null +++ b/server/internal/automation/eval_test.go @@ -0,0 +1,219 @@ +package automation + +import ( + "context" + "encoding/json" + "testing" +) + +// fakeState is an in-memory StateReader: issueID -> field -> value. +type fakeState map[string]map[string]string + +func (f fakeState) IssueField(_ context.Context, id, field string) (string, bool, error) { + m, ok := f[id] + if !ok { + return "", false, nil + } + v, ok := m[field] + return v, ok, nil +} + +func statusChangedEvent(subjectID, from, to string) EventView { + return EventView{ + Type: "issue.status_changed", + SubjectID: subjectID, + ActorType: "member", + ActorID: uuidM, + Payload: map[string]any{"from": from, "to": to}, + } +} + +func TestEvaluateEventTypeMismatch(t *testing.T) { + ev, err := Evaluate(context.Background(), + EventView{Type: "comment.created"}, + EvalRevision{EventType: "issue.status_changed", FireMode: FirePerEvent}, + fakeState{}) + if err != nil { + t.Fatal(err) + } + if ev.Reason != ReasonEventTypeMismatch || ev.Matched || ev.Eligible { + t.Fatalf("unexpected: %+v", ev) + } +} + +func TestEvaluateMatchClauses(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done","subject_id":{"in":["` + uuidA + `"]},"from":{"exists":true}}`), + FireMode: FirePerEvent, + } + ev, err := Evaluate(context.Background(), event, rev, fakeState{}) + if err != nil { + t.Fatal(err) + } + if !ev.Matched || !ev.Eligible || ev.DecisionComplete || ev.Reason != ReasonMatched { + t.Fatalf("expected match, got %+v", ev) + } + if len(ev.MatchClauses) != 3 { + t.Errorf("clauses = %d, want 3", len(ev.MatchClauses)) + } + + // A wrong `to` value fails the match. + rev.Match = json.RawMessage(`{"to":"blocked"}`) + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.Matched || ev.Reason != ReasonNoMatch { + t.Errorf("expected no_match, got %+v", ev) + } + + // exists:false on a present field fails. + rev.Match = json.RawMessage(`{"to":{"exists":false}}`) + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.Matched { + t.Errorf("exists:false on a present field should not match") + } +} + +func TestEvaluateConditionsAgainstCurrentState(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + } + // Both done → conditions met. + state := fakeState{uuidA: {"status": "done"}, uuidB: {"status": "done"}} + ev, err := Evaluate(context.Background(), event, rev, state) + if err != nil { + t.Fatal(err) + } + if !ev.ConditionsMet || ev.Reason != ReasonMatched || ev.EvaluatedAgainst != EvaluatedAgainstCurrentState { + t.Fatalf("expected conditions met against current state, got %+v", ev) + } + + // B not done → all-condition fails. + state[uuidB] = map[string]string{"status": "todo"} + ev, _ = Evaluate(context.Background(), event, rev, state) + if ev.ConditionsMet || ev.Reason != ReasonConditionFalse || ev.Eligible { + t.Fatalf("expected condition_false, got %+v", ev) + } + + // A missing issue is treated as unsatisfied for `all`. + rev.Conditions = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidC}, All: "done"}}} + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.ConditionsMet { + t.Errorf("missing issue must not satisfy an all-condition") + } + + // `any` semantics: at least one match suffices. + rev.Conditions = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, Any: "done"}}} + ev, _ = Evaluate(context.Background(), event, rev, fakeState{uuidA: {"status": "done"}, uuidB: {"status": "todo"}}) + if !ev.ConditionsMet { + t.Errorf("any-condition should be met when one issue matches") + } +} + +func TestEvaluateIssueFieldCondition(t *testing.T) { + event := statusChangedEvent(uuidA, "todo", "in_progress") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssueField: &IssueFieldCond{ID: uuidA, Field: "assignee_id", Eq: uuidM}}}, + } + ev, _ := Evaluate(context.Background(), event, rev, fakeState{uuidA: {"assignee_id": uuidM}}) + if !ev.ConditionsMet { + t.Errorf("issue_field eq should match") + } + ev, _ = Evaluate(context.Background(), event, rev, fakeState{uuidA: {"assignee_id": uuidB}}) + if ev.ConditionsMet { + t.Errorf("issue_field eq should not match a different assignee") + } +} + +// The evaluator output must carry observed + expected + op + present so a matcher +// can store one condition/match snapshot without re-reading state (review point 2). +func TestEvaluateStructuredSnapshot(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + } + state := fakeState{uuidA: {"status": "done"}, uuidB: {"status": "todo"}} + ev, err := Evaluate(context.Background(), event, rev, state) + if err != nil { + t.Fatal(err) + } + + // The match clause records op + observed + expected. + if len(ev.MatchClauses) != 1 { + t.Fatalf("match clauses = %d, want 1", len(ev.MatchClauses)) + } + mc := ev.MatchClauses[0] + if mc.Field != "to" || mc.Op != string(MatchEq) || mc.Observed != "done" || !mc.Present || len(mc.Expected) != 1 || mc.Expected[0] != "done" || !mc.Matched { + t.Errorf("match clause not fully structured: %+v", mc) + } + + // The condition records mode + expected + per-issue observed status. + if len(ev.Conditions) != 1 { + t.Fatalf("conditions = %d, want 1", len(ev.Conditions)) + } + c := ev.Conditions[0] + if c.Kind != "issues_status" || c.Mode != "all" || len(c.Expected) != 1 || c.Expected[0] != "done" || len(c.Issues) != 2 { + t.Fatalf("condition not fully structured: %+v", c) + } + byID := map[string]IssueObserved{} + for _, io := range c.Issues { + byID[io.ID] = io + } + if a := byID[uuidA]; a.Observed != "done" || !a.Present || !a.Matched { + t.Errorf("issue A observed wrong: %+v", a) + } + if b := byID[uuidB]; b.Observed != "todo" || !b.Present || b.Matched { + t.Errorf("issue B observed wrong: %+v", b) + } + + // Snapshots serialize the same structured inputs the matcher will store. + ms, err := ev.MatchSnapshot() + if err != nil || len(ms) == 0 { + t.Fatalf("match snapshot: %v", err) + } + cs, err := ev.ConditionSnapshot() + if err != nil || len(cs) == 0 { + t.Fatalf("condition snapshot: %v", err) + } +} + +// ProjectPayload is the fail-closed redaction for the correlation debug view. +func TestProjectPayload(t *testing.T) { + p := ProjectPayload("issue.created", map[string]any{"status": "todo", "title": "secret", "priority": "high", "bogus": 1}) + if _, ok := p["title"]; ok { + t.Error("free-text title must be redacted") + } + if _, ok := p["bogus"]; ok { + t.Error("undeclared field must be dropped") + } + if p["status"] != "todo" || p["priority"] != "high" { + t.Errorf("declared fields dropped: %v", p) + } + if len(ProjectPayload("issue.exploded", map[string]any{"x": 1})) != 0 { + t.Error("an unknown event type must project to an empty payload (fail-closed)") + } +} + +func TestEvaluateRisingEdgeNote(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FireRisingEdge, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA}, All: "done"}}}, + } + ev, _ := Evaluate(context.Background(), event, rev, fakeState{uuidA: {"status": "done"}}) + if ev.Reason != ReasonMatched || ev.Note == "" { + t.Fatalf("rising_edge matched should carry a read-only latch note, got %+v", ev) + } +} diff --git a/server/internal/automation/schema.go b/server/internal/automation/schema.go new file mode 100644 index 00000000000..364e3a261df --- /dev/null +++ b/server/internal/automation/schema.go @@ -0,0 +1,207 @@ +package automation + +import "github.com/multica-ai/multica/server/internal/domainevent" + +// FieldKind classifies a matchable event field so the validator can range-check +// clause values (e.g. a uuid field rejects a non-uuid literal). +type FieldKind int + +const ( + FieldString FieldKind = iota + FieldUUID +) + +// EventSchema declares, for one domain event type, which envelope/payload field +// paths a hook may match on. The design (§5.3/§6) requires that "字段路径必须由 +// 对应 event schema 声明" — this registry is that declaration, kept in lockstep +// with the domainevent payload structs. Only user-triggerable events are listed; +// system-derived events (issue.stage_completed) are not user-authorable in v1. +type EventSchema struct { + Type string + MatchFields map[string]FieldKind +} + +// envelopeMatchFields are common to every event type (the domain_event envelope +// columns a hook may match on). subject_id is the primary join key; actor_* let +// a rule scope to who caused the event. +var envelopeMatchFields = map[string]FieldKind{ + "subject_id": FieldUUID, + "actor_type": FieldString, + "actor_id": FieldUUID, +} + +// eventSchemas is the authoritative match-field registry, one entry per +// user-authorable v1 event type. Payload fields mirror the domainevent payload +// structs' json tags exactly. +var eventSchemas = buildEventSchemas(map[string]map[string]FieldKind{ + domainevent.TypeIssueCreated: { + "status": FieldString, + "priority": FieldString, + "parent_issue_id": FieldUUID, + "assignee_type": FieldString, + "assignee_id": FieldUUID, + "origin_type": FieldString, + }, + domainevent.TypeIssueStatusChanged: { + "from": FieldString, + "to": FieldString, + }, + domainevent.TypeIssueAssigned: { + "from_assignee_type": FieldString, + "from_assignee_id": FieldUUID, + "to_assignee_type": FieldString, + "to_assignee_id": FieldUUID, + }, + domainevent.TypeCommentCreated: { + "issue_id": FieldUUID, + "author_type": FieldString, + "author_id": FieldUUID, + "parent_id": FieldUUID, + }, + domainevent.TypeTaskCompleted: { + "issue_id": FieldUUID, + "agent_id": FieldUUID, + }, + domainevent.TypeTaskFailed: { + "issue_id": FieldUUID, + "agent_id": FieldUUID, + "retry_eligible": FieldString, + "error_code": FieldString, + }, +}) + +func buildEventSchemas(payloadFields map[string]map[string]FieldKind) map[string]EventSchema { + out := make(map[string]EventSchema, len(payloadFields)) + for evtType, fields := range payloadFields { + merged := make(map[string]FieldKind, len(fields)+len(envelopeMatchFields)) + for k, v := range envelopeMatchFields { + merged[k] = v + } + for k, v := range fields { + merged[k] = v + } + out[evtType] = EventSchema{Type: evtType, MatchFields: merged} + } + return out +} + +// SchemaFor returns the match-field schema for a user-authorable event type. +func SchemaFor(eventType string) (EventSchema, bool) { + s, ok := eventSchemas[eventType] + return s, ok +} + +// PayloadFields returns the declared payload field names for an event type — its +// schema's match fields minus the common envelope fields. An unknown event type +// returns an empty set. Free-text / undeclared payload keys (e.g. an issue title) +// are intentionally absent, so a projection over this set is fail-closed. +func PayloadFields(eventType string) map[string]bool { + schema, ok := eventSchemas[eventType] + if !ok { + return map[string]bool{} + } + out := make(map[string]bool, len(schema.MatchFields)) + for f := range schema.MatchFields { + if _, isEnvelope := envelopeMatchFields[f]; !isEnvelope { + out[f] = true + } + } + return out +} + +// ProjectPayload returns a copy of payload keeping ONLY the event type's declared +// payload fields (fail-closed redaction, §10): undeclared, sensitive or free-text +// keys are dropped, and an unknown event type yields an empty object. Used to +// project a domain event's payload for the read-only correlation debug surface. +func ProjectPayload(eventType string, payload map[string]any) map[string]any { + allowed := PayloadFields(eventType) + out := make(map[string]any, len(allowed)) + for k, v := range payload { + if allowed[k] { + out[k] = v + } + } + return out +} + +// Action types. User actions are creatable through the public API; system-only +// actions are reserved for managed system hooks (PR5) and are rejected on a +// user-authored spec. +const ( + ActionSetIssueStatus = "set_issue_status" + ActionTriggerAgent = "trigger_agent" + ActionAddComment = "add_comment" + ActionSendInbox = "send_inbox" + ActionRunAutopilot = "run_autopilot" + + ActionSetIssueStatusMany = "set_issue_status_many" // system-only + ActionTriggerIssueAssignee = "trigger_issue_assignee" // system-only +) + +// userActionTypes is the set of action types a user-authored hook may use. +var userActionTypes = map[string]bool{ + ActionSetIssueStatus: true, + ActionTriggerAgent: true, + ActionAddComment: true, + ActionSendInbox: true, + ActionRunAutopilot: true, +} + +// systemActionTypes is the set of action types reserved for managed system +// hooks; a user spec that names one is rejected (§8 — user hooks do not inherit +// the system routing bypass). +var systemActionTypes = map[string]bool{ + ActionSetIssueStatusMany: true, + ActionTriggerIssueAssignee: true, +} + +// issue field names valid in an issue_field condition (§5.4). +const ( + IssueFieldStatus = "status" + IssueFieldAssigneeID = "assignee_id" + IssueFieldParentIssueID = "parent_issue_id" +) + +var validIssueFields = map[string]bool{ + IssueFieldStatus: true, + IssueFieldAssigneeID: true, + IssueFieldParentIssueID: true, +} + +// validIssueStatuses mirrors the issue.status DB CHECK (migrations/001) and the +// handler's validIssueStatuses. A status-valued condition or action is rejected +// unless its value is one of these, so a hook can never persist an unreachable +// status (MUL-4332 PR2 review point 3). +var validIssueStatuses = map[string]bool{ + "backlog": true, + "todo": true, + "in_progress": true, + "in_review": true, + "done": true, + "blocked": true, + "cancelled": true, +} + +// isValidIssueStatus reports whether s is a persistable issue status. +func isValidIssueStatus(s string) bool { return validIssueStatuses[s] } + +// conditionDependencyEvent maps a condition to the single v1 domain event that +// can change its truth value. A rising_edge hook must listen to exactly that +// event so its latch can be re-evaluated (§5.2). In the v1 fixed vocabulary +// each condition depends on exactly one event type. +func conditionDependencyEvent(c ConditionSpec) (string, bool) { + switch { + case c.IssuesStatus != nil: + return domainevent.TypeIssueStatusChanged, true + case c.IssueField != nil: + switch c.IssueField.Field { + case IssueFieldStatus: + return domainevent.TypeIssueStatusChanged, true + case IssueFieldAssigneeID: + return domainevent.TypeIssueAssigned, true + case IssueFieldParentIssueID: + return domainevent.TypeIssueCreated, true + } + } + return "", false +} diff --git a/server/internal/automation/spec.go b/server/internal/automation/spec.go new file mode 100644 index 00000000000..8064f0657f2 --- /dev/null +++ b/server/internal/automation/spec.go @@ -0,0 +1,193 @@ +// Package automation implements the Event Hooks MVP policy layer (issue +// MUL-4332): the user-authored hook specification, the versioned event schema +// registry every rule binds against, and the typed validator that rejects an +// illegal rule at the API boundary rather than deep inside a worker. +// +// This package is pure policy/config: it parses and validates hook specs and +// declares the fixed vocabulary of events, conditions and actions. It performs +// NO matching and NO execution — the durable matcher/executor is a later slice +// (PR3+) and will reuse the parsed spec and the schema registry defined here. +package automation + +import ( + "encoding/json" + "fmt" +) + +// Fire modes (hook_revision.fire_mode). +const ( + FirePerEvent = "per_event" + FireRisingEdge = "rising_edge" +) + +// Scope types (hook.scope_type). An issue-scoped hook is owned by / displayed +// on an issue; it does not implicitly restrict the event subject — that is the +// job of `when`. +const ( + ScopeWorkspace = "workspace" + ScopeIssue = "issue" +) + +// HookSpec is the user-authored hook definition — the POST/PATCH request body. +// It maps onto two rows: the hook (name, scope) and its immutable revision +// (event_type, match, conditions, fire_mode, actions). +type HookSpec struct { + Name string `json:"name"` + Scope *ScopeSpec `json:"scope,omitempty"` + When WhenSpec `json:"when"` + If []ConditionSpec `json:"if,omitempty"` + Fire FireSpec `json:"fire"` + Do []ActionSpec `json:"do"` +} + +// ScopeSpec is the hook lifecycle owner. Absent means workspace scope. +type ScopeSpec struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` +} + +// WhenSpec selects the triggering event and an optional set of match clauses on +// its envelope/payload fields. Match is kept raw here and parsed/validated by +// ParseMatch against the event's schema so the wire form and the typed model +// stay decoupled. +type WhenSpec struct { + Event string `json:"event"` + Match json.RawMessage `json:"match,omitempty"` +} + +// FireSpec selects per_event or rising_edge semantics. +type FireSpec struct { + Mode string `json:"mode"` +} + +// ConditionSpec is one predicate over current workspace state. Exactly one of +// the fixed-vocabulary variants must be set (§5.4). +type ConditionSpec struct { + IssuesStatus *IssuesStatusCond `json:"issues_status,omitempty"` + IssueField *IssueFieldCond `json:"issue_field,omitempty"` +} + +// IssuesStatusCond asserts every (all) or any of the given issues is in a +// status. Exactly one of All/Any is set. +type IssuesStatusCond struct { + IDs []string `json:"ids"` + All string `json:"all,omitempty"` + Any string `json:"any,omitempty"` +} + +// IssueFieldCond asserts a single issue field equals / is in a set of values. +type IssueFieldCond struct { + ID string `json:"id"` + Field string `json:"field"` + Eq string `json:"eq,omitempty"` + In []string `json:"in,omitempty"` +} + +// ActionSpec is one action in the ordered `do` list. Fields are a superset over +// all action types; the validator enforces the required/allowed set per type. +// Parameters are literals in v1 (event-field binding and templating land with +// the executor slice). +type ActionSpec struct { + Type string `json:"type"` + IssueID string `json:"issue_id,omitempty"` + Status string `json:"status,omitempty"` + AgentID string `json:"agent_id,omitempty"` + MemberID string `json:"member_id,omitempty"` + Message string `json:"message,omitempty"` + AutopilotID string `json:"autopilot_id,omitempty"` +} + +// MatchOp is the operator of a single match clause. +type MatchOp string + +const ( + MatchEq MatchOp = "eq" + MatchIn MatchOp = "in" + MatchExists MatchOp = "exists" +) + +// MatchClause is one parsed match predicate on a single event field path. +type MatchClause struct { + Op MatchOp + Value string // for eq + Set []string // for in + Exists bool // for exists +} + +// Match is the parsed set of clauses keyed by event field path. +type Match map[string]MatchClause + +// ParseMatch decodes the raw `when.match` object into typed clauses. A value is +// either a JSON scalar (→ eq), an object {"in": [...]} (→ in) or an object +// {"exists": true|false} (→ exists). It does NOT check field paths against a +// schema — that is Validator.validateMatch's job — so it can be reused by the +// executor once the field set is already known-valid. +func ParseMatch(raw json.RawMessage) (Match, error) { + if len(raw) == 0 { + return Match{}, nil + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, fmt.Errorf("match must be an object: %w", err) + } + out := make(Match, len(obj)) + for field, rawVal := range obj { + clause, err := parseMatchClause(rawVal) + if err != nil { + return nil, fmt.Errorf("match.%s: %w", field, err) + } + out[field] = clause + } + return out, nil +} + +func parseMatchClause(raw json.RawMessage) (MatchClause, error) { + // Try an object form first: {"in": [...]} or {"exists": bool}. + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err == nil { + switch { + case len(obj) != 1: + return MatchClause{}, fmt.Errorf("clause object must have exactly one of \"in\" or \"exists\"") + case obj["in"] != nil: + var set []string + if err := json.Unmarshal(obj["in"], &set); err != nil { + return MatchClause{}, fmt.Errorf("\"in\" must be a string array: %w", err) + } + if len(set) == 0 { + return MatchClause{}, fmt.Errorf("\"in\" must not be empty") + } + return MatchClause{Op: MatchIn, Set: set}, nil + case obj["exists"] != nil: + var exists bool + if err := json.Unmarshal(obj["exists"], &exists); err != nil { + return MatchClause{}, fmt.Errorf("\"exists\" must be a boolean: %w", err) + } + return MatchClause{Op: MatchExists, Exists: exists}, nil + default: + return MatchClause{}, fmt.Errorf("unsupported clause; use a scalar, {\"in\": [...]} or {\"exists\": bool}") + } + } + // Otherwise a scalar equality. Accept string/number/bool, normalised to string. + var scalar any + if err := json.Unmarshal(raw, &scalar); err != nil { + return MatchClause{}, fmt.Errorf("value must be a scalar or clause object: %w", err) + } + switch v := scalar.(type) { + case string: + return MatchClause{Op: MatchEq, Value: v}, nil + case bool: + return MatchClause{Op: MatchEq, Value: fmt.Sprintf("%t", v)}, nil + case float64: + return MatchClause{Op: MatchEq, Value: formatNumber(v)}, nil + default: + return MatchClause{}, fmt.Errorf("equality value must be a string, number or boolean") + } +} + +func formatNumber(f float64) string { + // Integers render without a trailing .0 so a match on an int field reads naturally. + if f == float64(int64(f)) { + return fmt.Sprintf("%d", int64(f)) + } + return fmt.Sprintf("%g", f) +} diff --git a/server/internal/automation/validate.go b/server/internal/automation/validate.go new file mode 100644 index 00000000000..61b911bf5ef --- /dev/null +++ b/server/internal/automation/validate.go @@ -0,0 +1,364 @@ +package automation + +import ( + "errors" + "fmt" + + "github.com/multica-ai/multica/server/internal/util" +) + +// Guardrail limits (§9). Rate/concurrency/depth limits belong to the executor +// slice; these are the shape limits enforced at author time. +const ( + MaxActionsPerHook = 8 + MaxNameLength = 200 + MaxConditionIDs = 100 + MaxMatchSetSize = 100 + MaxMessageLength = 4000 +) + +// ValidationError is a user-fixable problem with a hook spec. Handlers map it to +// HTTP 400 (never 500) so a bad rule can never reach a worker (§13). +type ValidationError struct{ msg string } + +func (e *ValidationError) Error() string { return e.msg } + +// NewValidationError builds a ValidationError. Exported so callers that extend +// author-time validation with checks this pure package cannot do (e.g. the +// service's workspace-scoped target existence checks) surface the same typed +// error and share the handler's single 400 mapping. +func NewValidationError(format string, args ...any) *ValidationError { + return &ValidationError{msg: fmt.Sprintf(format, args...)} +} + +func verr(format string, args ...any) error { + return &ValidationError{msg: fmt.Sprintf(format, args...)} +} + +// AsValidationError reports whether err is a ValidationError. +func AsValidationError(err error) (*ValidationError, bool) { + var ve *ValidationError + if errors.As(err, &ve) { + return ve, true + } + return nil, false +} + +// Validate performs complete typed validation of a user-authored hook spec: +// event schema, match fields, condition dependency, action schema, fire-mode +// coverage and shape limits. It is the single author-time gate reused by both +// the create and patch paths. +func Validate(spec HookSpec) error { + if spec.Name == "" { + return verr("name is required") + } + if len(spec.Name) > MaxNameLength { + return verr("name must be at most %d characters", MaxNameLength) + } + if err := validateScope(spec.Scope); err != nil { + return err + } + + schema, ok := SchemaFor(spec.When.Event) + if !ok { + return verr("unknown or non-authorable event type %q", spec.When.Event) + } + if err := validateMatch(spec.When.Match, schema); err != nil { + return err + } + + for i, cond := range spec.If { + if err := validateCondition(cond); err != nil { + return verr("if[%d]: %s", i, err.Error()) + } + } + + if err := validateFire(spec); err != nil { + return err + } + + if len(spec.Do) == 0 { + return verr("do must contain at least one action") + } + if len(spec.Do) > MaxActionsPerHook { + return verr("do must contain at most %d actions", MaxActionsPerHook) + } + for i, action := range spec.Do { + if err := validateAction(action); err != nil { + return verr("do[%d]: %s", i, err.Error()) + } + } + return nil +} + +func validateScope(scope *ScopeSpec) error { + if scope == nil { + return nil + } + switch scope.Type { + case ScopeWorkspace: + if scope.ID != "" { + return verr("scope.id must be empty for a workspace-scoped hook") + } + case ScopeIssue: + if !validUUID(scope.ID) { + return verr("scope.id must be a valid issue id for an issue-scoped hook") + } + default: + return verr("scope.type must be %q or %q", ScopeWorkspace, ScopeIssue) + } + return nil +} + +func validateMatch(raw []byte, schema EventSchema) error { + match, err := ParseMatch(raw) + if err != nil { + return verr("%s", err.Error()) + } + for field, clause := range match { + kind, ok := schema.MatchFields[field] + if !ok { + return verr("match field %q is not declared by event %q", field, schema.Type) + } + if err := validateClauseValues(field, kind, clause); err != nil { + return err + } + } + return nil +} + +func validateClauseValues(field string, kind FieldKind, clause MatchClause) error { + if clause.Op == MatchIn && len(clause.Set) > MaxMatchSetSize { + return verr("match field %q has %d values, at most %d allowed", field, len(clause.Set), MaxMatchSetSize) + } + if kind != FieldUUID { + return nil // string fields accept any scalar; existence needs no value check + } + switch clause.Op { + case MatchEq: + if !validUUID(clause.Value) { + return verr("match field %q expects a uuid, got %q", field, clause.Value) + } + case MatchIn: + for _, v := range clause.Set { + if !validUUID(v) { + return verr("match field %q expects uuids, got %q", field, v) + } + } + } + return nil +} + +func validateCondition(c ConditionSpec) error { + set := 0 + if c.IssuesStatus != nil { + set++ + } + if c.IssueField != nil { + set++ + } + if set != 1 { + return verr("exactly one of issues_status or issue_field must be set") + } + if c.IssuesStatus != nil { + return validateIssuesStatus(*c.IssuesStatus) + } + return validateIssueField(*c.IssueField) +} + +func validateIssuesStatus(c IssuesStatusCond) error { + if len(c.IDs) == 0 { + return verr("issues_status.ids must not be empty") + } + if len(c.IDs) > MaxConditionIDs { + return verr("issues_status.ids has %d ids, at most %d allowed", len(c.IDs), MaxConditionIDs) + } + for _, id := range c.IDs { + if !validUUID(id) { + return verr("issues_status.ids must be uuids, got %q", id) + } + } + hasAll, hasAny := c.All != "", c.Any != "" + if hasAll == hasAny { + return verr("exactly one of issues_status.all or issues_status.any must be set") + } + status := c.All + if hasAny { + status = c.Any + } + if !isValidIssueStatus(status) { + return verr("issues_status status %q is not a valid issue status", status) + } + return nil +} + +func validateIssueField(c IssueFieldCond) error { + if !validUUID(c.ID) { + return verr("issue_field.id must be a uuid") + } + if !validIssueFields[c.Field] { + return verr("issue_field.field must be one of status, assignee_id, parent_issue_id") + } + hasEq, hasIn := c.Eq != "", len(c.In) > 0 + if hasEq == hasIn { + return verr("exactly one of issue_field.eq or issue_field.in must be set") + } + if len(c.In) > MaxConditionIDs { + return verr("issue_field.in has %d values, at most %d allowed", len(c.In), MaxConditionIDs) + } + switch c.Field { + case IssueFieldStatus: + // status is a free string but must be a persistable issue status. + for _, v := range collectValues(c.Eq, c.In) { + if !isValidIssueStatus(v) { + return verr("issue_field status %q is not a valid issue status", v) + } + } + default: + // id-shaped fields (assignee_id / parent_issue_id) require uuids. + if hasEq && !validUUID(c.Eq) { + return verr("issue_field.eq must be a uuid for field %q", c.Field) + } + for _, v := range c.In { + if !validUUID(v) { + return verr("issue_field.in must be uuids for field %q", c.Field) + } + } + } + return nil +} + +// collectValues returns eq (if set) plus the in slice as one list. +func collectValues(eq string, in []string) []string { + out := make([]string, 0, len(in)+1) + if eq != "" { + out = append(out, eq) + } + out = append(out, in...) + return out +} + +func validateFire(spec HookSpec) error { + switch spec.Fire.Mode { + case FirePerEvent: + return nil + case FireRisingEdge: + return validateRisingEdgeCoverage(spec) + default: + return verr("fire.mode must be %q or %q", FirePerEvent, FireRisingEdge) + } +} + +// validateRisingEdgeCoverage enforces §5.2: a rising_edge hook's latch can only +// be re-evaluated by the event it listens to, so every condition must depend on +// exactly the hook's own event type, and there must be at least one condition to +// gate on. This is the extractable dependency check the design requires at save +// time for the v1 fixed vocabulary. +func validateRisingEdgeCoverage(spec HookSpec) error { + if len(spec.If) == 0 { + return verr("rising_edge requires at least one condition in if") + } + for i, cond := range spec.If { + dep, ok := conditionDependencyEvent(cond) + if !ok { + return verr("if[%d]: condition has no known change event, cannot use rising_edge", i) + } + if dep != spec.When.Event { + return verr("rising_edge hook must listen to %q so its condition in if[%d] can be re-evaluated, but when.event is %q", dep, i, spec.When.Event) + } + } + return nil +} + +// actionAllowedFields declares the exact set of ActionSpec parameter fields each +// action type may set. Any other non-empty field is rejected so a stray param +// (e.g. an agent_id smuggled onto add_comment) can never be persisted onto a +// revision (MUL-4332 PR2 review point 3). +var actionAllowedFields = map[string]map[string]bool{ + ActionSetIssueStatus: {"issue_id": true, "status": true}, + ActionTriggerAgent: {"issue_id": true, "agent_id": true}, + ActionAddComment: {"issue_id": true, "message": true}, + ActionSendInbox: {"member_id": true, "message": true}, + ActionRunAutopilot: {"autopilot_id": true}, +} + +func validateAction(a ActionSpec) error { + if systemActionTypes[a.Type] { + return verr("action type %q is reserved for system hooks", a.Type) + } + if !userActionTypes[a.Type] { + return verr("unknown action type %q", a.Type) + } + if err := rejectUnexpectedActionFields(a); err != nil { + return err + } + if a.Message != "" && len(a.Message) > MaxMessageLength { + return verr("%s message must be at most %d characters", a.Type, MaxMessageLength) + } + switch a.Type { + case ActionSetIssueStatus: + if !validUUID(a.IssueID) { + return verr("set_issue_status requires a valid issue_id") + } + if a.Status == "" { + return verr("set_issue_status requires status") + } + if !isValidIssueStatus(a.Status) { + return verr("set_issue_status status %q is not a valid issue status", a.Status) + } + case ActionTriggerAgent: + if !validUUID(a.IssueID) { + return verr("trigger_agent requires a valid issue_id") + } + if !validUUID(a.AgentID) { + return verr("trigger_agent requires a valid agent_id") + } + case ActionAddComment: + if !validUUID(a.IssueID) { + return verr("add_comment requires a valid issue_id") + } + if a.Message == "" { + return verr("add_comment requires message") + } + case ActionSendInbox: + if !validUUID(a.MemberID) { + return verr("send_inbox requires a valid member_id") + } + if a.Message == "" { + return verr("send_inbox requires message") + } + case ActionRunAutopilot: + if !validUUID(a.AutopilotID) { + return verr("run_autopilot requires a valid autopilot_id") + } + } + return nil +} + +// rejectUnexpectedActionFields fails if the action sets any parameter field not +// allowed for its type — strict "exactly the allowed fields" enforcement. +func rejectUnexpectedActionFields(a ActionSpec) error { + allowed := actionAllowedFields[a.Type] + present := map[string]string{ + "issue_id": a.IssueID, + "status": a.Status, + "agent_id": a.AgentID, + "member_id": a.MemberID, + "message": a.Message, + "autopilot_id": a.AutopilotID, + } + for field, value := range present { + if value != "" && !allowed[field] { + return verr("%s does not accept field %q", a.Type, field) + } + } + return nil +} + +func validUUID(s string) bool { + if s == "" { + return false + } + _, err := util.ParseUUID(s) + return err == nil +} diff --git a/server/internal/automation/validate_test.go b/server/internal/automation/validate_test.go new file mode 100644 index 00000000000..ca8258eadee --- /dev/null +++ b/server/internal/automation/validate_test.go @@ -0,0 +1,177 @@ +package automation + +import ( + "encoding/json" + "strings" + "testing" +) + +const ( + uuidA = "11111111-1111-1111-1111-111111111111" + uuidB = "22222222-2222-2222-2222-222222222222" + uuidC = "33333333-3333-3333-3333-333333333333" + uuidM = "44444444-4444-4444-4444-444444444444" +) + +// validPerEvent is the minimal accepted spec: comment.created → add a comment. +func validPerEvent() HookSpec { + return HookSpec{ + Name: "notify on comment", + When: WhenSpec{Event: "comment.created"}, + Fire: FireSpec{Mode: FirePerEvent}, + Do: []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "hi"}}, + } +} + +// validRisingEdge is the design's A2 join: A and B both done → start C + inbox. +func validRisingEdge() HookSpec { + return HookSpec{ + Name: "A/B done then start C", + When: WhenSpec{Event: "issue.status_changed", Match: json.RawMessage(`{"subject_id":{"in":["` + uuidA + `","` + uuidB + `"]}}`)}, + If: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + Fire: FireSpec{Mode: FireRisingEdge}, + Do: []ActionSpec{ + {Type: ActionSetIssueStatus, IssueID: uuidC, Status: "todo"}, + {Type: ActionSendInbox, MemberID: uuidM, Message: "A/B done, C started"}, + }, + } +} + +func TestValidateAcceptsValidSpecs(t *testing.T) { + for name, spec := range map[string]HookSpec{ + "per_event minimal": validPerEvent(), + "rising_edge join": validRisingEdge(), + } { + if err := Validate(spec); err != nil { + t.Errorf("%s: expected valid, got %v", name, err) + } + } +} + +func TestValidateRejectsInvalidSpecs(t *testing.T) { + cases := []struct { + name string + mutate func(*HookSpec) + wantSub string // substring the error message must contain + }{ + {"missing name", func(s *HookSpec) { s.Name = "" }, "name is required"}, + {"unknown event", func(s *HookSpec) { s.When.Event = "issue.exploded" }, "non-authorable event"}, + {"system event not authorable", func(s *HookSpec) { s.When.Event = "issue.stage_completed" }, "non-authorable event"}, + {"undeclared match field", func(s *HookSpec) { + s.When = WhenSpec{Event: "comment.created", Match: json.RawMessage(`{"nope":"x"}`)} + }, "not declared"}, + {"uuid match field non-uuid", func(s *HookSpec) { + s.When = WhenSpec{Event: "comment.created", Match: json.RawMessage(`{"issue_id":"not-a-uuid"}`)} + }, "expects a uuid"}, + {"empty do", func(s *HookSpec) { s.Do = nil }, "at least one action"}, + {"too many actions", func(s *HookSpec) { + s.Do = make([]ActionSpec, MaxActionsPerHook+1) + for i := range s.Do { + s.Do[i] = ActionSpec{Type: ActionAddComment, IssueID: uuidC, Message: "x"} + } + }, "at most"}, + {"system-only action", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatusMany}} + }, "reserved for system"}, + {"unknown action", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: "delete_workspace"}} + }, "unknown action type"}, + {"set_issue_status missing status", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatus, IssueID: uuidC}} + }, "requires status"}, + {"set_issue_status invalid status enum", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatus, IssueID: uuidC, Status: "ascended"}} + }, "not a valid issue status"}, + {"add_comment with disallowed agent_id field", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "hi", AgentID: uuidA}} + }, "does not accept field"}, + {"issues_status invalid status enum", func(s *HookSpec) { + s.When = WhenSpec{Event: "issue.status_changed"} + s.Fire = FireSpec{Mode: FirePerEvent} + s.If = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA}, All: "ascended"}}} + }, "not a valid issue status"}, + {"trigger_agent bad agent", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionTriggerAgent, IssueID: uuidC, AgentID: "x"}} + }, "valid agent_id"}, + {"bad fire mode", func(s *HookSpec) { s.Fire.Mode = "always" }, "fire.mode"}, + {"scope issue without id", func(s *HookSpec) { + s.Scope = &ScopeSpec{Type: ScopeIssue} + }, "issue-scoped"}, + {"scope workspace with id", func(s *HookSpec) { + s.Scope = &ScopeSpec{Type: ScopeWorkspace, ID: uuidA} + }, "must be empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + spec := validPerEvent() + tc.mutate(&spec) + err := Validate(spec) + if err == nil { + t.Fatalf("expected rejection, got nil") + } + if _, ok := AsValidationError(err); !ok { + t.Fatalf("expected *ValidationError, got %T", err) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub) + } + }) + } +} + +// The rising-edge dependency check (§5.2) is subtle enough to test on its own. +func TestValidateRisingEdgeCoverage(t *testing.T) { + t.Run("requires a condition", func(t *testing.T) { + spec := validRisingEdge() + spec.If = nil + if err := Validate(spec); err == nil || !strings.Contains(err.Error(), "at least one condition") { + t.Fatalf("rising_edge without conditions must be rejected, got %v", err) + } + }) + t.Run("must listen to the condition's change event", func(t *testing.T) { + spec := validRisingEdge() + // issues_status depends on issue.status_changed; listening to comment.created + // means the latch could never be re-evaluated. + spec.When = WhenSpec{Event: "comment.created"} + err := Validate(spec) + if err == nil || !strings.Contains(err.Error(), "rising_edge hook must listen to") { + t.Fatalf("rising_edge listening to the wrong event must be rejected, got %v", err) + } + }) + t.Run("assignee condition binds to issue.assigned", func(t *testing.T) { + spec := HookSpec{ + Name: "assignee gate", + When: WhenSpec{Event: "issue.assigned"}, + If: []ConditionSpec{{IssueField: &IssueFieldCond{ID: uuidC, Field: IssueFieldAssigneeID, Eq: uuidM}}}, + Fire: FireSpec{Mode: FireRisingEdge}, + Do: []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "assigned"}}, + } + if err := Validate(spec); err != nil { + t.Fatalf("assignee rising_edge on issue.assigned should be valid, got %v", err) + } + }) +} + +// ParseMatch is the shared wire→typed step; verify each clause shape. +func TestParseMatch(t *testing.T) { + m, err := ParseMatch(json.RawMessage(`{"to":"done","subject_id":{"in":["` + uuidA + `"]},"actor_id":{"exists":true}}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := m["to"]; got.Op != MatchEq || got.Value != "done" { + t.Errorf("to clause = %+v, want eq done", got) + } + if got := m["subject_id"]; got.Op != MatchIn || len(got.Set) != 1 || got.Set[0] != uuidA { + t.Errorf("subject_id clause = %+v, want in [%s]", got, uuidA) + } + if got := m["actor_id"]; got.Op != MatchExists || !got.Exists { + t.Errorf("actor_id clause = %+v, want exists true", got) + } + + if _, err := ParseMatch(json.RawMessage(`{"to":{"in":[]}}`)); err == nil { + t.Errorf("empty in set must be rejected") + } + if _, err := ParseMatch(json.RawMessage(`"notanobject"`)); err == nil { + t.Errorf("non-object match must be rejected") + } +} diff --git a/server/internal/daemon/client.go b/server/internal/daemon/client.go index 196866f65c9..f14ca179722 100644 --- a/server/internal/daemon/client.go +++ b/server/internal/daemon/client.go @@ -421,11 +421,57 @@ func (c *Client) PinTaskSession(ctx context.Context, taskID, sessionID, workDir return c.postJSON(ctx, fmt.Sprintf("/api/daemon/tasks/%s/session", taskID), body, nil) } -// RecoverOrphans tells the server to fail any dispatched/running tasks the -// previous daemon process for this runtime left behind. The server will -// auto-retry eligible tasks. +// maxRecoverOrphanPages bounds the drain loop below. Each page fails up to +// orphanRecoveryBatchSize (500) rows, so this covers ~500k orphaned rows per +// registration — far beyond any real restart. It is purely a backstop so a buggy +// server response (has_more stuck true) can never spin forever; any residual is +// reclaimed by the next registration. +const maxRecoverOrphanPages = 1000 + +// RecoverOrphans tells the server to fail any dispatched/running tasks the previous +// daemon process for this runtime left behind, and to auto-retry the eligible ones. +// +// It DRAINS the runtime's orphans across pages (MUL-4332 review round 3, point 1): +// registration has already upserted the runtime back to `online`, so the server's +// every-tick offline sweep will not reap anything a single capped call leaves +// behind. We therefore loop, threading the server's keyset cursor, until the server +// reports no more pages. The cursor advances over poison (unresolvable) rows the +// server skips, so a page of poison at the front cannot stall the drain. func (c *Client) RecoverOrphans(ctx context.Context, runtimeID string) error { - return c.postJSON(ctx, fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID), map[string]any{}, nil) + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + // paginate:true is our capability signal (MUL-4332 review round 4, point 2): it + // tells the server this client will drive the drain itself, so the server hands + // back one page at a time. A server that predates the flag ignores it and does + // its own single-shot recovery (handled via io.EOF below); a legacy daemon that + // lacks the flag is drained entirely server-side. + body := map[string]any{"paginate": true} + for page := 0; page < maxRecoverOrphanPages; page++ { + var resp struct { + HasMore bool `json:"has_more"` + NextCursorCreatedAt string `json:"next_cursor_created_at"` + NextCursorID string `json:"next_cursor_id"` + } + if err := c.postJSON(ctx, path, body, &resp); err != nil { + // An empty 200 body (an older server that predates paging, or a proxy) + // decodes to io.EOF. Treat it as a single non-paged recovery: there is + // no cursor to continue with, so stop cleanly rather than erroring. + if errors.Is(err, io.EOF) { + return nil + } + return err + } + // Stop on the last page, or defensively if the server signalled more but + // gave us no cursor to advance (never expected — avoids re-draining page 0). + if !resp.HasMore || resp.NextCursorCreatedAt == "" || resp.NextCursorID == "" { + return nil + } + body = map[string]any{ + "paginate": true, + "cursor_created_at": resp.NextCursorCreatedAt, + "cursor_id": resp.NextCursorID, + } + } + return nil } // GetTaskStatus returns the current status of a task. Used by the daemon to diff --git a/server/internal/domainevent/domainevent.go b/server/internal/domainevent/domainevent.go new file mode 100644 index 00000000000..2b8d43e995e --- /dev/null +++ b/server/internal/domainevent/domainevent.go @@ -0,0 +1,120 @@ +// Package domainevent is the transactional-outbox event layer for the Event +// Hooks MVP (MUL-4332). It defines the versioned v1 domain event catalog and a +// tx-aware writer that persists one immutable row into the `domain_event` table +// IN THE SAME TRANSACTION as the domain fact that produced it. +// +// The contract is deliberately narrow: +// +// - A caller that already writes a domain fact inside a pgx.Tx obtains the +// tx-bound *db.Queries (via Queries.WithTx) and calls Write(ctx, qtx, evt) +// before committing. Fact and event commit atomically — a crash between +// them is impossible, which is what makes the outbox durable. +// - A caller whose write is a bare autocommit statement uses WriteInTx, which +// wraps the write + event in one transaction. +// +// PR1 has NO consumer: rows land dispatch_status='pending' and nothing reads +// them, so wiring Write into a domain path is a zero-behavior-change addition. +// The matcher/executor that claims pending rows arrives in PR3. +// +// This package is intentionally separate from internal/events (the in-memory +// events.Bus). The Bus stays best-effort, post-commit, and serves realtime UI; +// domain_event is the durable, transactional source of truth for automation. +package domainevent + +import "github.com/jackc/pgx/v5/pgtype" + +// Event type names (the `type` column). Dotted noun.verb, distinct from the +// colon-delimited events.Bus protocol names so the two namespaces never blur. +const ( + TypeIssueCreated = "issue.created" + TypeIssueStatusChanged = "issue.status_changed" + TypeIssueAssigned = "issue.assigned" + TypeCommentCreated = "comment.created" + TypeTaskCompleted = "task.completed" + TypeTaskFailed = "task.failed" + + // TypeIssueStageCompleted is a derived sensor event emitted by the PR5 + // stage frontier sensor, not by any v1 domain write. The constant is + // declared here so the catalog is complete and validators recognise it. + TypeIssueStageCompleted = "issue.stage_completed" +) + +// Subject types (the `subject_type` column): what entity the event is about. +const ( + SubjectIssue = "issue" + SubjectComment = "comment" + SubjectTask = "task" +) + +// Actor types (the `actor_type` column): who caused the event. +const ( + ActorMember = "member" + ActorAgent = "agent" + ActorSystem = "system" + ActorHook = "hook" +) + +// Dispatch statuses (the `dispatch_status` column). Only DispatchPending is +// produced in PR1; the rest are advanced by the PR3 matcher/executor. +const ( + DispatchPending = "pending" + DispatchDispatching = "dispatching" + DispatchDispatched = "dispatched" + DispatchFailed = "failed" +) + +// typeSpec pins the invariants of one event type so Write can reject a +// malformed envelope before it reaches the DB. +type typeSpec struct { + subject string + version int32 +} + +// catalog is the authoritative v1 registry. schema_version is 1 for every type +// in v1; a breaking payload change bumps the version here and the payload's +// schemaVersion() together. +var catalog = map[string]typeSpec{ + TypeIssueCreated: {subject: SubjectIssue, version: 1}, + TypeIssueStatusChanged: {subject: SubjectIssue, version: 1}, + TypeIssueAssigned: {subject: SubjectIssue, version: 1}, + TypeCommentCreated: {subject: SubjectComment, version: 1}, + TypeTaskCompleted: {subject: SubjectTask, version: 1}, + TypeTaskFailed: {subject: SubjectTask, version: 1}, + TypeIssueStageCompleted: {subject: SubjectIssue, version: 1}, +} + +var validActorTypes = map[string]bool{ + ActorMember: true, + ActorAgent: true, + ActorSystem: true, + ActorHook: true, +} + +// Actor identifies who caused an event. The ID is invalid (NULL) for a system +// actor, which has no member/agent identity. +type Actor struct { + Type string + ID pgtype.UUID +} + +// MemberActor / AgentActor / HookActor / SystemActor build an Actor from an +// identity the call site already holds. +func MemberActor(id pgtype.UUID) Actor { return Actor{Type: ActorMember, ID: id} } +func AgentActor(id pgtype.UUID) Actor { return Actor{Type: ActorAgent, ID: id} } +func HookActor(id pgtype.UUID) Actor { return Actor{Type: ActorHook, ID: id} } +func SystemActor() Actor { return Actor{Type: ActorSystem} } + +// ActorFrom builds an Actor from a raw (type, id) pair — for call sites that +// carry an existing creator_type/creator_id or author_type/author_id. +// +// It is fail-closed (MUL-4332 review point 6): a system actor is normalised to +// carry no id, but an unknown type is NOT silently degraded to system — it is +// passed through unchanged so Event.validate rejects it and the transaction +// aborts, rather than permanently recording a mislabelled actor as system. +func ActorFrom(actorType string, id pgtype.UUID) Actor { + if actorType == ActorSystem { + // A system actor never carries an id; drop a stray one. + return SystemActor() + } + return Actor{Type: actorType, ID: id} +} diff --git a/server/internal/domainevent/event.go b/server/internal/domainevent/event.go new file mode 100644 index 00000000000..21a48ad844c --- /dev/null +++ b/server/internal/domainevent/event.go @@ -0,0 +1,224 @@ +package domainevent + +import ( + "encoding/json" + "fmt" + + "github.com/jackc/pgx/v5/pgtype" +) + +// Event is a fully-formed, validated-on-write domain event ready to persist. +// Construct it through a typed constructor (IssueStatusChanged, CommentCreated, +// …) rather than by hand so Type / SubjectType / SchemaVersion / Payload always +// agree with the catalog. +// +// A zero CorrelationID marks a root event: Write assigns correlation_id = id and +// hop_count = 0. The Causation* / HopCount fields stay zero for every v1 domain +// write (only the PR3 executor, replaying a reaction, sets them). +type Event struct { + WorkspaceID pgtype.UUID + Type string + SchemaVersion int32 + SubjectType string + SubjectID pgtype.UUID + ActorType string + ActorID pgtype.UUID + Payload []byte + + CorrelationID pgtype.UUID + CausationExecutionID pgtype.UUID + CausationActionIndex pgtype.Int4 + HopCount int32 + + // buildErr carries a payload-marshal failure from the constructor so call + // sites stay error-free; Write surfaces it and aborts the transaction. + buildErr error +} + +// payload is implemented by every typed payload struct so a single generic +// builder can stamp the envelope fields from the payload itself. +type payload interface { + eventType() string + subjectType() string + schemaVersion() int32 +} + +func newEvent(workspaceID, subjectID pgtype.UUID, actor Actor, p payload) Event { + raw, err := json.Marshal(p) + return Event{ + WorkspaceID: workspaceID, + Type: p.eventType(), + SchemaVersion: p.schemaVersion(), + SubjectType: p.subjectType(), + SubjectID: subjectID, + ActorType: actor.Type, + ActorID: actor.ID, + Payload: raw, + buildErr: err, + } +} + +// validate rejects an envelope that disagrees with the catalog before it hits +// the DB. It is a safety net beneath the typed constructors, and the guard the +// future public REST/CLI create path will reuse. +func (e Event) validate() error { + if e.buildErr != nil { + return fmt.Errorf("domainevent: marshal payload: %w", e.buildErr) + } + spec, ok := catalog[e.Type] + if !ok { + return fmt.Errorf("domainevent: unknown event type %q", e.Type) + } + if e.SchemaVersion != spec.version { + return fmt.Errorf("domainevent: %s schema_version %d, want %d", e.Type, e.SchemaVersion, spec.version) + } + if e.SubjectType != spec.subject { + return fmt.Errorf("domainevent: %s subject_type %q, want %q", e.Type, e.SubjectType, spec.subject) + } + if !validActorTypes[e.ActorType] { + return fmt.Errorf("domainevent: invalid actor_type %q", e.ActorType) + } + // Fail-closed actor identity (MUL-4332 review point 6): a system actor must + // carry NO id, and every other actor type must carry a valid one — so a + // dropped / unparsable id can never be silently recorded as a null or + // system actor. The caller's transaction aborts instead. + if e.ActorType == ActorSystem { + if e.ActorID.Valid { + return fmt.Errorf("domainevent: %s: system actor must not carry an actor_id", e.Type) + } + } else if !e.ActorID.Valid { + return fmt.Errorf("domainevent: %s: %s actor requires a valid actor_id", e.Type, e.ActorType) + } + if !e.WorkspaceID.Valid { + return fmt.Errorf("domainevent: %s missing workspace_id", e.Type) + } + if !e.SubjectID.Valid { + return fmt.Errorf("domainevent: %s missing subject_id", e.Type) + } + if len(e.Payload) == 0 { + return fmt.Errorf("domainevent: %s empty payload", e.Type) + } + return nil +} + +// ---- v1 payloads ---------------------------------------------------------- +// +// UUID-valued fields are JSON strings (empty + omitempty when absent). Call +// sites convert a pgtype.UUID with util.UUIDToString, which yields "" for an +// invalid/NULL value — so an absent parent/assignee is omitted, not null. + +// IssueCreatedPayload — subject is the new issue. +type IssueCreatedPayload struct { + Status string `json:"status"` + Title string `json:"title"` + Priority string `json:"priority,omitempty"` + ParentIssueID string `json:"parent_issue_id,omitempty"` + AssigneeType string `json:"assignee_type,omitempty"` + AssigneeID string `json:"assignee_id,omitempty"` + OriginType string `json:"origin_type,omitempty"` +} + +func (IssueCreatedPayload) eventType() string { return TypeIssueCreated } +func (IssueCreatedPayload) subjectType() string { return SubjectIssue } +func (IssueCreatedPayload) schemaVersion() int32 { return 1 } + +// IssueStatusChangedPayload — subject is the issue whose status moved. +type IssueStatusChangedPayload struct { + From string `json:"from"` + To string `json:"to"` +} + +func (IssueStatusChangedPayload) eventType() string { return TypeIssueStatusChanged } +func (IssueStatusChangedPayload) subjectType() string { return SubjectIssue } +func (IssueStatusChangedPayload) schemaVersion() int32 { return 1 } + +// IssueAssignedPayload — subject is the issue whose assignee changed. +type IssueAssignedPayload struct { + FromAssigneeType string `json:"from_assignee_type,omitempty"` + FromAssigneeID string `json:"from_assignee_id,omitempty"` + ToAssigneeType string `json:"to_assignee_type,omitempty"` + ToAssigneeID string `json:"to_assignee_id,omitempty"` +} + +func (IssueAssignedPayload) eventType() string { return TypeIssueAssigned } +func (IssueAssignedPayload) subjectType() string { return SubjectIssue } +func (IssueAssignedPayload) schemaVersion() int32 { return 1 } + +// CommentCreatedPayload — subject is the comment; issue_id locates its thread. +type CommentCreatedPayload struct { + IssueID string `json:"issue_id"` + AuthorType string `json:"author_type"` + AuthorID string `json:"author_id,omitempty"` + ParentID string `json:"parent_id,omitempty"` +} + +func (CommentCreatedPayload) eventType() string { return TypeCommentCreated } +func (CommentCreatedPayload) subjectType() string { return SubjectComment } +func (CommentCreatedPayload) schemaVersion() int32 { return 1 } + +// TaskCompletedPayload — subject is the task; issue_id/agent_id locate it. +type TaskCompletedPayload struct { + IssueID string `json:"issue_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` +} + +func (TaskCompletedPayload) eventType() string { return TypeTaskCompleted } +func (TaskCompletedPayload) subjectType() string { return SubjectTask } +func (TaskCompletedPayload) schemaVersion() int32 { return 1 } + +// TaskFailedPayload — subject is the task. RetryEligible is an atomically-decidable +// fact: at the instant this failure committed, the task satisfied the auto-retry +// eligibility predicate (an infra-shaped reason, still within the attempt budget, +// not an autopilot run, and reporting to an issue or chat). Every path derives it +// from the shared retryEligible predicate in the SAME transaction as the fail + its +// event, so the flag can never contradict the eligibility decision. +// +// It deliberately does NOT promise that a retry child was — or ever will be — +// created. The single FailTask path does create the child in this same transaction, +// but the bulk sweeper / orphan-recovery paths create it best-effort AFTER commit, +// where CreateRetryTask can return an error or the process can crash between the two +// steps. So a consumer must treat task.failed as terminal for the subject task and +// read retry_eligible only as "this failure was retry-eligible", never as "a fresh +// attempt is guaranteed to arrive". +type TaskFailedPayload struct { + IssueID string `json:"issue_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + RetryEligible bool `json:"retry_eligible"` + ErrorCode string `json:"error_code,omitempty"` +} + +func (TaskFailedPayload) eventType() string { return TypeTaskFailed } +func (TaskFailedPayload) subjectType() string { return SubjectTask } +func (TaskFailedPayload) schemaVersion() int32 { return 1 } + +// ---- typed constructors --------------------------------------------------- + +// IssueCreated builds an issue.created event for the given new issue. +func IssueCreated(workspaceID, issueID pgtype.UUID, actor Actor, p IssueCreatedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// IssueStatusChanged builds an issue.status_changed event. +func IssueStatusChanged(workspaceID, issueID pgtype.UUID, actor Actor, p IssueStatusChangedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// IssueAssigned builds an issue.assigned event. +func IssueAssigned(workspaceID, issueID pgtype.UUID, actor Actor, p IssueAssignedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// CommentCreated builds a comment.created event; subjectID is the comment id. +func CommentCreated(workspaceID, commentID pgtype.UUID, actor Actor, p CommentCreatedPayload) Event { + return newEvent(workspaceID, commentID, actor, p) +} + +// TaskCompleted builds a task.completed event; subjectID is the task id. +func TaskCompleted(workspaceID, taskID pgtype.UUID, actor Actor, p TaskCompletedPayload) Event { + return newEvent(workspaceID, taskID, actor, p) +} + +// TaskFailed builds a task.failed event; subjectID is the task id. +func TaskFailed(workspaceID, taskID pgtype.UUID, actor Actor, p TaskFailedPayload) Event { + return newEvent(workspaceID, taskID, actor, p) +} diff --git a/server/internal/domainevent/event_test.go b/server/internal/domainevent/event_test.go new file mode 100644 index 00000000000..3e6e9d80481 --- /dev/null +++ b/server/internal/domainevent/event_test.go @@ -0,0 +1,153 @@ +package domainevent + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +func testUUID(t *testing.T) pgtype.UUID { + t.Helper() + return pgtype.UUID{Bytes: uuid.New(), Valid: true} +} + +// Every typed constructor must produce an envelope that agrees with the catalog +// (type, subject_type, schema_version) and passes validate — this is the guard +// the future public create path reuses, so it must hold for the internal callers +// too. +func TestConstructorsProduceValidEnvelopes(t *testing.T) { + ws := testUUID(t) + subj := testUUID(t) + actor := MemberActor(testUUID(t)) + + cases := []struct { + name string + evt Event + wantType string + wantSubject string + }{ + {"issue.created", IssueCreated(ws, subj, actor, IssueCreatedPayload{Status: "todo", Title: "x"}), TypeIssueCreated, SubjectIssue}, + {"issue.status_changed", IssueStatusChanged(ws, subj, actor, IssueStatusChangedPayload{From: "todo", To: "done"}), TypeIssueStatusChanged, SubjectIssue}, + {"issue.assigned", IssueAssigned(ws, subj, actor, IssueAssignedPayload{ToAssigneeType: "agent"}), TypeIssueAssigned, SubjectIssue}, + {"comment.created", CommentCreated(ws, subj, actor, CommentCreatedPayload{IssueID: uuid.NewString(), AuthorType: "member"}), TypeCommentCreated, SubjectComment}, + {"task.completed", TaskCompleted(ws, subj, SystemActor(), TaskCompletedPayload{IssueID: uuid.NewString()}), TypeTaskCompleted, SubjectTask}, + {"task.failed", TaskFailed(ws, subj, SystemActor(), TaskFailedPayload{RetryEligible: true}), TypeTaskFailed, SubjectTask}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if err := tc.evt.validate(); err != nil { + t.Fatalf("validate: %v", err) + } + if tc.evt.Type != tc.wantType { + t.Errorf("Type = %q, want %q", tc.evt.Type, tc.wantType) + } + if tc.evt.SubjectType != tc.wantSubject { + t.Errorf("SubjectType = %q, want %q", tc.evt.SubjectType, tc.wantSubject) + } + if tc.evt.SchemaVersion != 1 { + t.Errorf("SchemaVersion = %d, want 1", tc.evt.SchemaVersion) + } + if !json.Valid(tc.evt.Payload) { + t.Errorf("payload is not valid JSON: %s", tc.evt.Payload) + } + // A v1 domain write is always a root event. + if tc.evt.CorrelationID.Valid || tc.evt.HopCount != 0 { + t.Errorf("expected root event (no correlation, hop 0), got correlation.Valid=%v hop=%d", tc.evt.CorrelationID.Valid, tc.evt.HopCount) + } + }) + } +} + +func TestValidateRejectsBadEnvelopes(t *testing.T) { + ws := testUUID(t) + subj := testUUID(t) + base := IssueStatusChanged(ws, subj, MemberActor(testUUID(t)), IssueStatusChangedPayload{From: "a", To: "b"}) + + mutate := func(f func(*Event)) Event { + e := base + f(&e) + return e + } + + cases := map[string]Event{ + "unknown type": mutate(func(e *Event) { e.Type = "issue.exploded" }), + "wrong schema ver": mutate(func(e *Event) { e.SchemaVersion = 2 }), + "wrong subject type": mutate(func(e *Event) { e.SubjectType = SubjectTask }), + "bad actor type": mutate(func(e *Event) { e.ActorType = "wizard" }), + "member without id": mutate(func(e *Event) { e.ActorType = ActorMember; e.ActorID = pgtype.UUID{} }), + "agent without id": mutate(func(e *Event) { e.ActorType = ActorAgent; e.ActorID = pgtype.UUID{} }), + "system with id": mutate(func(e *Event) { e.ActorType = ActorSystem; e.ActorID = testUUID(t) }), + "unknown type w/ id": mutate(func(e *Event) { e.ActorType = "wizard"; e.ActorID = testUUID(t) }), + "missing workspace": mutate(func(e *Event) { e.WorkspaceID = pgtype.UUID{} }), + "missing subject": mutate(func(e *Event) { e.SubjectID = pgtype.UUID{} }), + "empty payload": mutate(func(e *Event) { e.Payload = nil }), + } + for name, e := range cases { + t.Run(name, func(t *testing.T) { + if err := e.validate(); err == nil { + t.Fatalf("expected validate to reject %s", name) + } + }) + } +} + +// ActorFrom is fail-closed (MUL-4332 review point 6): a system actor is +// normalised to carry no id, but an unknown/empty type is passed through +// unchanged so validate rejects it — it must NOT be silently laundered into a +// system actor, which would permanently mis-record authorship. +func TestActorFromFailClosed(t *testing.T) { + id := testUUID(t) + ws, subj := testUUID(t), testUUID(t) + build := func(a Actor) error { + return IssueStatusChanged(ws, subj, a, IssueStatusChangedPayload{From: "a", To: "b"}).validate() + } + + if a := ActorFrom("member", id); a.Type != ActorMember || a.ID != id { + t.Errorf("member: got %+v", a) + } + if a := ActorFrom("system", id); a.Type != ActorSystem || a.ID.Valid { + t.Errorf("system actor never carries an id, got %+v", a) + } + // Unknown / empty types survive into the envelope and are rejected there. + if a := ActorFrom("wizard", id); a.Type != "wizard" { + t.Errorf("unknown type must pass through, got %+v", a) + } + if err := build(ActorFrom("wizard", id)); err == nil { + t.Error("unknown actor type must fail validate, not degrade to system") + } + if err := build(ActorFrom("", id)); err == nil { + t.Error("empty actor type must fail validate") + } + // The valid shapes still pass. + if err := build(MemberActor(id)); err != nil { + t.Errorf("member actor should validate: %v", err) + } + if err := build(SystemActor()); err != nil { + t.Errorf("system actor should validate: %v", err) + } +} + +// The payload JSON shape is a wire contract the PR3 matcher's fixed-vocabulary +// predicates bind against, so pin the exact keys. +func TestPayloadJSONShape(t *testing.T) { + raw, err := json.Marshal(IssueStatusChangedPayload{From: "todo", To: "in_progress"}) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m["from"] != "todo" || m["to"] != "in_progress" { + t.Errorf("unexpected status payload: %s", raw) + } + + // omitempty must drop absent optional UUID fields rather than emit null. + raw, _ = json.Marshal(IssueCreatedPayload{Status: "todo", Title: "x"}) + if got := string(raw); got != `{"status":"todo","title":"x"}` { + t.Errorf("expected optional keys omitted, got %s", got) + } +} diff --git a/server/internal/domainevent/writer.go b/server/internal/domainevent/writer.go new file mode 100644 index 00000000000..99e933d1aaf --- /dev/null +++ b/server/internal/domainevent/writer.go @@ -0,0 +1,124 @@ +package domainevent + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// IssueCreatedFromRow builds an issue.created event from a freshly-created issue +// row, stamping its birth assignee/origin into the payload. Shared by every +// issue producer (HTTP create, autopilot dispatch, onboarding) so the event +// payload stays identical regardless of which path created the issue. +func IssueCreatedFromRow(issue db.Issue) Event { + return IssueCreated(issue.WorkspaceID, issue.ID, + ActorFrom(issue.CreatorType, issue.CreatorID), + IssueCreatedPayload{ + Status: issue.Status, + Title: issue.Title, + Priority: issue.Priority, + ParentIssueID: util.UUIDToString(issue.ParentIssueID), + AssigneeType: issue.AssigneeType.String, + AssigneeID: util.UUIDToString(issue.AssigneeID), + OriginType: issue.OriginType.String, + }) +} + +// Creator is the single DB method Write needs. *db.Queries satisfies it, so a +// caller passes the tx-bound handle it already holds (base.WithTx(tx)). Keeping +// it an interface (not *db.Queries) lets tests substitute a fake. +type Creator interface { + CreateDomainEvent(ctx context.Context, arg db.CreateDomainEventParams) (db.DomainEvent, error) +} + +// txBeginner is the subset of *pgxpool.Pool that WriteInTx needs. +type txBeginner interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + +// Write validates evt and inserts it as a pending outbox row using q, which MUST +// be bound to the caller's transaction (Queries.WithTx) so the event commits +// atomically with the domain fact. It returns the persisted row (seq/created_at +// populated by the DB). +// +// For a root event (CorrelationID unset — every v1 domain write) Write assigns a +// fresh id and sets correlation_id = id, hop_count = 0. +func Write(ctx context.Context, q Creator, evt Event) (db.DomainEvent, error) { + if err := evt.validate(); err != nil { + return db.DomainEvent{}, err + } + + id := pgUUID(uuid.New()) + correlation := evt.CorrelationID + if !correlation.Valid { + // Root event: it is its own correlation head. + correlation = id + } + + row, err := q.CreateDomainEvent(ctx, db.CreateDomainEventParams{ + ID: id, + WorkspaceID: evt.WorkspaceID, + Type: evt.Type, + SchemaVersion: evt.SchemaVersion, + SubjectType: evt.SubjectType, + SubjectID: evt.SubjectID, + ActorType: evt.ActorType, + ActorID: evt.ActorID, + Payload: evt.Payload, + CorrelationID: correlation, + CausationExecutionID: evt.CausationExecutionID, + CausationActionIndex: evt.CausationActionIndex, + HopCount: evt.HopCount, + }) + if err != nil { + return db.DomainEvent{}, fmt.Errorf("domainevent: insert %s: %w", evt.Type, err) + } + return row, nil +} + +// WriteInTx wraps a domain write that is otherwise a bare autocommit statement. +// It opens a transaction, hands fn a tx-bound *db.Queries to perform the write, +// then persists the events fn returns — all in one commit, so the fact and its +// events land atomically or not at all. +// +// err := domainevent.WriteInTx(ctx, pool, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { +// row, err := qtx.UpdateIssueStatus(ctx, params) +// if err != nil { return nil, err } +// return []domainevent.Event{domainevent.IssueStatusChanged(ws, row.ID, actor, ...)}, nil +// }) +// +// base is the pool-bound *db.Queries; WriteInTx rebinds it to the new tx. If fn +// returns no events (e.g. the write turned out to be a no-op), the transaction +// still commits the fn's own writes. +func WriteInTx(ctx context.Context, tb txBeginner, base *db.Queries, fn func(qtx *db.Queries) ([]Event, error)) error { + tx, err := tb.Begin(ctx) + if err != nil { + return fmt.Errorf("domainevent: begin tx: %w", err) + } + defer tx.Rollback(ctx) + + qtx := base.WithTx(tx) + events, err := fn(qtx) + if err != nil { + return err + } + for _, evt := range events { + if _, err := Write(ctx, qtx, evt); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("domainevent: commit tx: %w", err) + } + return nil +} + +func pgUUID(u uuid.UUID) pgtype.UUID { + return pgtype.UUID{Bytes: u, Valid: true} +} diff --git a/server/internal/domainevent/writer_db_test.go b/server/internal/domainevent/writer_db_test.go new file mode 100644 index 00000000000..d8c72c26743 --- /dev/null +++ b/server/internal/domainevent/writer_db_test.go @@ -0,0 +1,204 @@ +package domainevent + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// newDomainEventTestPool connects to the shared, already-migrated test DB. Like +// every other DB-backed test in this repo it SKIPS (never fails) when no +// database is reachable — the schema is migrated out of band by `make test`/CI. +func newDomainEventTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + dbURL = "postgres://multica:multica@localhost:5432/multica?sslmode=disable" + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + pool, err := pgxpool.New(ctx, dbURL) + if err != nil { + t.Skipf("database unavailable: %v", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + t.Skipf("database unreachable: %v", err) + } + // Confirm the migration is applied; a DB pinned to an older schema should + // skip, not fail with a missing-relation error. + if _, err := pool.Exec(ctx, "SELECT 1 FROM domain_event LIMIT 0"); err != nil { + pool.Close() + t.Skipf("domain_event table missing (migrate up first): %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func countEventsForWorkspace(t *testing.T, pool *pgxpool.Pool, ws pgtype.UUID) int { + t.Helper() + var n int + if err := pool.QueryRow(context.Background(), + `SELECT count(*) FROM domain_event WHERE workspace_id = $1`, ws).Scan(&n); err != nil { + t.Fatalf("count: %v", err) + } + return n +} + +// cleanupWorkspaceEvents deletes rows with context.Background so it survives a +// cancelled test context. +func cleanupWorkspaceEvents(pool *pgxpool.Pool, ws pgtype.UUID) { + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE workspace_id = $1`, ws) +} + +func standInFactEvent(ws pgtype.UUID) Event { + return IssueCreated(ws, pgUUID(uuid.New()), SystemActor(), IssueCreatedPayload{Status: "todo", Title: "stand-in fact"}) +} + +// A committed Write must persist exactly one row and stamp the root-event +// invariants: dispatch_status='pending', hop_count=0, correlation_id=id, a +// monotonic seq, and the exact payload. +func TestWriteCommitPersistsRootEvent(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + ws := pgUUID(uuid.New()) + subj := pgUUID(uuid.New()) + actor := MemberActor(pgUUID(uuid.New())) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + + evt := IssueStatusChanged(ws, subj, actor, IssueStatusChangedPayload{From: "todo", To: "done"}) + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + row, err := Write(ctx, queries.WithTx(tx), evt) + if err != nil { + tx.Rollback(ctx) + t.Fatalf("write: %v", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit: %v", err) + } + + if got := countEventsForWorkspace(t, pool, ws); got != 1 { + t.Fatalf("expected exactly 1 event, got %d", got) + } + if row.DispatchStatus != DispatchPending { + t.Errorf("dispatch_status = %q, want %q", row.DispatchStatus, DispatchPending) + } + if row.HopCount != 0 { + t.Errorf("hop_count = %d, want 0", row.HopCount) + } + if row.CorrelationID != row.ID { + t.Errorf("root event correlation_id (%v) must equal id (%v)", row.CorrelationID, row.ID) + } + if row.Seq <= 0 { + t.Errorf("seq should be a positive monotonic value, got %d", row.Seq) + } + if row.Type != TypeIssueStatusChanged { + t.Errorf("type = %q", row.Type) + } +} + +// The outbox durability invariant (MUL-4332 §1 kill test #1): a crash after the +// domain write but before commit must leave NO event — the write and the event +// share one transaction, so an uncommitted event simply does not exist. +func TestWriteRollbackPersistsNothing(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := Write(ctx, queries.WithTx(tx), standInFactEvent(ws)); err != nil { + tx.Rollback(ctx) + t.Fatalf("write: %v", err) + } + // Simulate the process dying before commit. + if err := tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("rolled-back event must not persist, found %d rows", got) + } +} + +// WriteInTx must be all-or-nothing: if the caller's domain write or its own +// event insert fails, the fact written inside fn rolls back too. +func TestWriteInTxAtomicity(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + t.Run("commit persists fact and event", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + // Stand-in "domain write" inside the tx. + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + return []Event{standInFactEvent(ws)}, nil + }) + if err != nil { + t.Fatalf("WriteInTx: %v", err) + } + if got := countEventsForWorkspace(t, pool, ws); got != 2 { + t.Fatalf("expected fact + event = 2 rows, got %d", got) + } + }) + + t.Run("fn error rolls back the fact", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + boom := errors.New("domain write failed") + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + return nil, boom + }) + if !errors.Is(err, boom) { + t.Fatalf("expected boom, got %v", err) + } + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("fn error must roll back the fact, found %d rows", got) + } + }) + + t.Run("invalid event rolls back the fact", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + bad := standInFactEvent(ws) + bad.Type = "issue.exploded" // fails validate in Write + return []Event{bad}, nil + }) + if err == nil { + t.Fatal("expected invalid event to fail WriteInTx") + } + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("invalid event must roll back the fact, found %d rows", got) + } + }) +} diff --git a/server/internal/featureflags/keys.go b/server/internal/featureflags/keys.go index 5d1f78e0848..613591d23aa 100644 --- a/server/internal/featureflags/keys.go +++ b/server/internal/featureflags/keys.go @@ -23,6 +23,20 @@ const ( // key as enabled so installed v0.4.0 desktop clients, which still gate the // switch on this config decision, receive the permanently enabled behavior. agentSkillTogglesCompat = "agents_skill_toggles" + // EventHooks gates the Event Hooks engine (MUL-4332). PR1 only lands the + // transactional-outbox event layer, which is always-on and consumer-less; + // this flag stays off until the PR3 matcher/executor ships so no reaction + // ever fires from a partially-built engine. Server-only, default off. + EventHooks = "automation_event_hooks" + // EventHookExecution gates ACTION EXECUTION specifically, separately from + // EventHooks. The two are deliberately distinct rollout stages: EventHooks + // alone opens the policy API, dry-run/explain and the matcher, which only + // record queued/skipped decisions, so a workspace can run the engine in shadow + // and inspect what it *would* do. Only this second switch lets the executor + // claim those queued executions and perform real side effects, so turning the + // engine on for shadow evaluation can never start mutating data. + // Server-only, default off, and required IN ADDITION to EventHooks. + EventHookExecution = "automation_event_hook_execution" ) var frontendPublicFlags = []string{ @@ -43,6 +57,20 @@ func ResourceLabelsEnabled(ctx context.Context, flags *featureflag.Service) bool return flags.IsEnabled(ctx, ResourceLabels, false) } +// EventHooksEnabled reports whether the Event Hooks engine may run reactions. +// PR1 does not consult it (the outbox writer is always-on); it exists so the +// PR3 matcher/executor can gate execution behind a default-off switch. +func EventHooksEnabled(ctx context.Context, flags *featureflag.Service) bool { + return flags.IsEnabled(ctx, EventHooks, false) +} + +// EventHookExecutionEnabled reports whether the executor may run real actions. It +// requires BOTH switches: the engine as a whole, and execution specifically. A +// workspace that has only enabled EventHooks stays in shadow mode. +func EventHookExecutionEnabled(ctx context.Context, flags *featureflag.Service) bool { + return EventHooksEnabled(ctx, flags) && flags.IsEnabled(ctx, EventHookExecution, false) +} + func EvaluateFrontendPublicFlags(ctx context.Context, flags *featureflag.Service) map[string]bool { out := make(map[string]bool, len(frontendPublicFlags)+1) for _, key := range frontendPublicFlags { diff --git a/server/internal/handler/agent_access.go b/server/internal/handler/agent_access.go index 53968c122ef..8fdd7493f36 100644 --- a/server/internal/handler/agent_access.go +++ b/server/internal/handler/agent_access.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/admission" "github.com/multica-ai/multica/server/internal/util" db "github.com/multica-ai/multica/server/pkg/db/generated" ) @@ -146,17 +147,7 @@ func (h *Handler) canAccessPrivateAgent(ctx context.Context, agent db.Agent, act // view gate and the ListAgents batch filter. A workspace target admits any // member; a member target admits the matching user; team targets are inert. func memberHitsInvocationTargets(targets []db.AgentInvocationTarget, userID string) bool { - for _, t := range targets { - switch t.TargetType { - case "workspace": - return true - case "member": - if uuidToString(t.TargetID) == userID { - return true - } - } - } - return false + return admission.MemberHitsInvocationTargets(targets, userID) } // memberAllowedToViewAgent is the ListAgents / aggregation filter predicate. diff --git a/server/internal/handler/autopilot.go b/server/internal/handler/autopilot.go index a157bcfa043..50af94c62a4 100644 --- a/server/internal/handler/autopilot.go +++ b/server/internal/handler/autopilot.go @@ -13,6 +13,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/admission" "github.com/multica-ai/multica/server/internal/analytics" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" "github.com/multica-ai/multica/server/internal/service" @@ -536,10 +537,7 @@ func (h *Handler) loadAutopilotInWorkspace(w http.ResponseWriter, r *http.Reques // write access. Explicit collaborator grants (memberCanWriteAutopilot) layer // on top of this (MUL-3807). func autopilotWriteByOwnership(ap db.Autopilot, member db.Member) bool { - if roleAllowed(member.Role, "owner", "admin") { - return true - } - return ap.CreatedByType == "member" && uuidToString(ap.CreatedByID) == uuidToString(member.UserID) + return admission.AutopilotWriteByOwnership(ap, member) } // memberCanWriteAutopilot reports whether the given member may perform write or diff --git a/server/internal/handler/comment.go b/server/internal/handler/comment.go index 5103460b770..d575e5566d3 100644 --- a/server/internal/handler/comment.go +++ b/server/internal/handler/comment.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/logger" "github.com/multica-ai/multica/server/internal/service" "github.com/multica-ai/multica/server/internal/util" @@ -1356,19 +1357,37 @@ func (h *Handler) CreateComment(w http.ResponseWriter, r *http.Request) { } } - comment, err := h.Queries.CreateComment(r.Context(), db.CreateCommentParams{ - IssueID: issue.ID, - WorkspaceID: issue.WorkspaceID, - AuthorType: authorType, - AuthorID: parseUUID(authorID), - Content: req.Content, - Type: req.Type, - ParentID: parentID, - SourceTaskID: sourceTaskID, - }) - if err != nil { - slog.Warn("create comment failed", append(logger.RequestAttrs(r), "error", err, "issue_id", issueID)...) - writeError(w, http.StatusInternalServerError, "failed to create comment: "+err.Error()) + // Transactional outbox (MUL-4332): the comment and its comment.created event + // commit in one transaction. subject is the comment; the payload locates its + // issue/thread so a hook can react without a second lookup. + commentActorUUID, _ := util.ParseUUID(authorID) + commentActor := domainevent.ActorFrom(authorType, commentActorUUID) + var comment db.Comment + if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, err := qtx.CreateComment(r.Context(), db.CreateCommentParams{ + IssueID: issue.ID, + WorkspaceID: issue.WorkspaceID, + AuthorType: authorType, + AuthorID: parseUUID(authorID), + Content: req.Content, + Type: req.Type, + ParentID: parentID, + SourceTaskID: sourceTaskID, + }) + if err != nil { + return nil, err + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, commentActor, + domainevent.CommentCreatedPayload{ + IssueID: uuidToString(created.IssueID), + AuthorType: authorType, + AuthorID: authorID, + ParentID: uuidToString(parentID), + })}, nil + }); writeErr != nil { + slog.Warn("create comment failed", append(logger.RequestAttrs(r), "error", writeErr, "issue_id", issueID)...) + writeError(w, http.StatusInternalServerError, "failed to create comment: "+writeErr.Error()) return } diff --git a/server/internal/handler/domain_event_outbox_test.go b/server/internal/handler/domain_event_outbox_test.go new file mode 100644 index 00000000000..5cf9888ce43 --- /dev/null +++ b/server/internal/handler/domain_event_outbox_test.go @@ -0,0 +1,192 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/multica-ai/multica/server/internal/domainevent" +) + +type outboxRow struct { + Type string + Payload []byte + ID string + CorrelationID string + DispatchStatus string + HopCount int32 +} + +// eventsForSubject returns every domain_event about a specific subject, ordered +// by seq. Scoping by subject_id (not the shared test workspace) keeps the read +// isolated from other tests running against the same DB. +func eventsForSubject(t *testing.T, subjectType, subjectID string) []outboxRow { + t.Helper() + rows, err := testPool.Query(context.Background(), + `SELECT type, payload, id::text, correlation_id::text, dispatch_status, hop_count + FROM domain_event + WHERE subject_type = $1 AND subject_id = $2 + ORDER BY seq`, subjectType, subjectID) + if err != nil { + t.Fatalf("query domain_event: %v", err) + } + defer rows.Close() + var out []outboxRow + for rows.Next() { + var r outboxRow + if err := rows.Scan(&r.Type, &r.Payload, &r.ID, &r.CorrelationID, &r.DispatchStatus, &r.HopCount); err != nil { + t.Fatalf("scan: %v", err) + } + out = append(out, r) + } + return out +} + +func payloadField(t *testing.T, raw []byte, key string) string { + t.Helper() + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + s, _ := m[key].(string) + return s +} + +// End-to-end: driving the real HTTP handlers must persist the transactional +// outbox events (MUL-4332). Proves issue.created / issue.status_changed / +// issue.assigned are written atomically by the create + update paths, with the +// root-event invariants (pending, hop 0, correlation = id) intact. +func TestOutboxEmittedByIssueHandlers(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + issueID := createTestIssue(t, "outbox e2e "+t.Name(), "todo", "none") + t.Cleanup(func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // 1) create → exactly one issue.created, a root event. + created := eventsForSubject(t, domainevent.SubjectIssue, issueID) + if len(created) != 1 { + t.Fatalf("expected 1 event after create, got %d (%+v)", len(created), created) + } + ev := created[0] + if ev.Type != domainevent.TypeIssueCreated { + t.Errorf("type = %q, want %q", ev.Type, domainevent.TypeIssueCreated) + } + if ev.DispatchStatus != domainevent.DispatchPending { + t.Errorf("dispatch_status = %q, want pending", ev.DispatchStatus) + } + if ev.HopCount != 0 { + t.Errorf("hop_count = %d, want 0", ev.HopCount) + } + if ev.CorrelationID != ev.ID { + t.Errorf("root correlation_id (%s) must equal id (%s)", ev.CorrelationID, ev.ID) + } + if got := payloadField(t, ev.Payload, "status"); got != "todo" { + t.Errorf("issue.created payload status = %q, want todo", got) + } + + // 2) update status + assignee in one call → status_changed + assigned. + uw := httptest.NewRecorder() + ureq := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{ + "status": "in_progress", + "assignee_type": "member", + "assignee_id": testUserID, + }) + ureq = withURLParam(ureq, "id", issueID) + testHandler.UpdateIssue(uw, ureq) + if uw.Code != http.StatusOK { + t.Fatalf("UpdateIssue: expected 200, got %d: %s", uw.Code, uw.Body.String()) + } + + all := eventsForSubject(t, domainevent.SubjectIssue, issueID) + var sawStatus, sawAssigned bool + for _, e := range all { + switch e.Type { + case domainevent.TypeIssueStatusChanged: + sawStatus = true + if from := payloadField(t, e.Payload, "from"); from != "todo" { + t.Errorf("status_changed from = %q, want todo", from) + } + if to := payloadField(t, e.Payload, "to"); to != "in_progress" { + t.Errorf("status_changed to = %q, want in_progress", to) + } + case domainevent.TypeIssueAssigned: + sawAssigned = true + if to := payloadField(t, e.Payload, "to_assignee_id"); to != testUserID { + t.Errorf("assigned to_assignee_id = %q, want %q", to, testUserID) + } + } + } + if !sawStatus { + t.Errorf("expected an issue.status_changed event, got %+v", all) + } + if !sawAssigned { + t.Errorf("expected an issue.assigned event, got %+v", all) + } +} + +// Two concurrent status transitions on the same issue must each record the TRUE +// edge, not both read the same pre-transition snapshot (MUL-4332 review point +// 3). Because the event `from` is now read under a row lock inside the update +// tx, the two updates serialize and their events chain (todo→A, A→B) instead of +// both reporting from="todo". +func TestOutboxStatusFromCorrectUnderConcurrency(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + issueID := createTestIssue(t, "outbox concurrency "+t.Name(), "todo", "none") + t.Cleanup(func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // Fire two different transitions concurrently. FOR UPDATE serializes them. + statuses := []string{"in_progress", "done"} + var wg sync.WaitGroup + for _, st := range statuses { + wg.Add(1) + go func(status string) { + defer wg.Done() + w := httptest.NewRecorder() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{"status": status}) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(w, req) + }(st) + } + wg.Wait() + + var changes []outboxRow + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, issueID) { + if e.Type == domainevent.TypeIssueStatusChanged { + changes = append(changes, e) + } + } + if len(changes) != 2 { + t.Fatalf("expected 2 status_changed events, got %d: %+v", len(changes), changes) + } + + // The froms must be distinct (the bug produced two from="todo"); one edge + // starts at "todo" and the other starts where the first landed (a valid + // chain), regardless of which transition won the lock first. + from0 := payloadField(t, changes[0].Payload, "from") + from1 := payloadField(t, changes[1].Payload, "from") + if from0 == from1 { + t.Fatalf("both events share from=%q — stale snapshot bug (review point 3): %+v", from0, changes) + } + // Identify the todo-rooted edge and assert the other edge chains off its `to`. + byFrom := map[string]outboxRow{from0: changes[0], from1: changes[1]} + todoEdge, ok := byFrom["todo"] + if !ok { + t.Fatalf("expected one edge to start at todo, got froms %q/%q", from0, from1) + } + firstTo := payloadField(t, todoEdge.Payload, "to") + if _, chained := byFrom[firstTo]; !chained { + t.Errorf("edges do not chain: todo→%s but no event starts from %s (%+v)", firstTo, firstTo, changes) + } +} diff --git a/server/internal/handler/domain_event_race_test.go b/server/internal/handler/domain_event_race_test.go new file mode 100644 index 00000000000..7977d3481be --- /dev/null +++ b/server/internal/handler/domain_event_race_test.go @@ -0,0 +1,134 @@ +package handler + +import ( + "context" + "fmt" + "net/http/httptest" + "sync" + "testing" + + "github.com/multica-ai/multica/server/internal/domainevent" +) + +// A status-only update that races a concurrent reassignment must not roll the +// assignee back to its pre-tx snapshot (MUL-4332 review point 1). UpdateIssue +// writes the nullable columns as bare narg, so before the fix a status-only +// write whose lock landed AFTER the reassignment committed would overwrite the +// assignee with the stale value it read before the tx — silently undoing the +// reassignment, and emitting no issue.assigned event to signal the reversal. +// After the fix every untouched column is rebuilt from the locked row, so the +// reassignment survives regardless of who wins the lock, and its assigned event +// is always present. Looped over fresh issues to exercise both interleavings. +func TestUpdateIssueConcurrentReassignNotRolledBack(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + const iterations = 6 + for i := 0; i < iterations; i++ { + issueID := createTestIssue(t, fmt.Sprintf("reassign-race %s #%d", t.Name(), i), "todo", "none") + func() { + defer func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }() + + var wg sync.WaitGroup + wg.Add(2) + // A: status-only update — must NOT touch the assignee. + go func() { + defer wg.Done() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{"status": "in_progress"}) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(httptest.NewRecorder(), req) + }() + // B: reassign to the member — must survive the concurrent status write. + go func() { + defer wg.Done() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{ + "assignee_type": "member", + "assignee_id": testUserID, + }) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(httptest.NewRecorder(), req) + }() + wg.Wait() + + var assigneeType, assigneeID string + if err := testPool.QueryRow(context.Background(), + `SELECT COALESCE(assignee_type, ''), COALESCE(assignee_id::text, '') FROM issue WHERE id = $1`, issueID). + Scan(&assigneeType, &assigneeID); err != nil { + t.Fatalf("iter %d: read issue assignee: %v", i, err) + } + if assigneeType != "member" || assigneeID != testUserID { + t.Fatalf("iter %d: assignee rolled back to (%q, %q), want (member, %s) — a status-only write clobbered the reassignment (review point 1)", + i, assigneeType, assigneeID, testUserID) + } + + var sawAssigned bool + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, issueID) { + if e.Type == domainevent.TypeIssueAssigned && payloadField(t, e.Payload, "to_assignee_id") == testUserID { + sawAssigned = true + } + } + if !sawAssigned { + t.Fatalf("iter %d: missing issue.assigned event for the reassignment", i) + } + }() + } +} + +// advanceIssueToDone (the merged-PR close path) must treat an already-done issue +// as a genuine no-op: the locked pre-image shows no transition, so it emits no +// event AND fires no parent child-done comment / realtime status_changed. A +// stale in_progress snapshot from a duplicate webhook delivery must not re-drive +// those side effects (MUL-4332 review point 4). +func TestAdvanceIssueToDoneNoOpSuppressesDuplicate(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + ctx := context.Background() + + parentID := createTestIssue(t, "no-op parent "+t.Name(), "in_progress", "none") + childID := createTestIssue(t, "no-op child "+t.Name(), "in_progress", "none") + // Link child to parent and complete it directly (bypassing the handler), so + // no child-done comment has fired yet and the parent starts clean. + if _, err := testPool.Exec(ctx, `UPDATE issue SET parent_issue_id = $1, status = 'done' WHERE id = $2`, parentID, childID); err != nil { + t.Fatalf("link + complete child: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM comment WHERE issue_id IN ($1, $2)`, parentID, childID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id IN ($1, $2)`, parentID, childID) + deleteTestIssue(t, childID) + deleteTestIssue(t, parentID) + }) + + childRow, err := testHandler.Queries.GetIssue(ctx, parseUUID(childID)) + if err != nil { + t.Fatalf("load child: %v", err) + } + // Simulate the stale webhook view: the child is already done in the DB, but + // the caller still holds an in_progress snapshot. + stale := childRow + stale.Status = "in_progress" + + before := issueCommentCount(t, parentID) + testHandler.advanceIssueToDone(ctx, stale, testWorkspaceID) + + if after := issueCommentCount(t, parentID); after != before { + t.Fatalf("parent comment count changed %d→%d — a no-op transition must not re-post the child-done comment (review point 4)", before, after) + } + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, childID) { + if e.Type == domainevent.TypeIssueStatusChanged { + t.Fatalf("a no-op advanceIssueToDone must emit no issue.status_changed event, got %+v", e) + } + } +} + +func issueCommentCount(t *testing.T, issueID string) int { + t.Helper() + var n int + if err := testPool.QueryRow(context.Background(), `SELECT count(*) FROM comment WHERE issue_id = $1`, issueID).Scan(&n); err != nil { + t.Fatalf("count comments: %v", err) + } + return n +} diff --git a/server/internal/handler/github.go b/server/internal/handler/github.go index 1f45430af77..22ea411d61d 100644 --- a/server/internal/handler/github.go +++ b/server/internal/handler/github.go @@ -23,6 +23,7 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/middleware" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" @@ -1361,32 +1362,67 @@ func (h *Handler) lookupIssueByIdentifier(ctx context.Context, workspaceID pgtyp } func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, workspaceID string) { - updated, err := h.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ - ID: issue.ID, - Status: "done", - WorkspaceID: issue.WorkspaceID, - }) - if err != nil { + // Transactional outbox (MUL-4332): a merged PR closing an issue is one of the + // most common status transitions to `done`, so emit issue.status_changed + // atomically with the status flip — but only on a real transition (an + // already-done issue produces no event). + var updated db.Issue + var before db.Issue + if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + // Lock + read the authoritative row so the event `from` is correct + // even if another writer moved the issue concurrently (review point 3). + locked, err := qtx.LockIssueRowForUpdate(ctx, db.LockIssueRowForUpdateParams{ + ID: issue.ID, + WorkspaceID: issue.WorkspaceID, + }) + if err != nil { + return nil, err + } + before = locked + u, err := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ + ID: issue.ID, + Status: "done", + WorkspaceID: issue.WorkspaceID, + }) + if err != nil { + return nil, err + } + updated = u + if before.Status == u.Status { + return nil, nil + } + return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), + domainevent.IssueStatusChangedPayload{From: before.Status, To: u.Status})}, nil + }); err != nil { slog.Warn("github: advance issue to done failed", "err", err) return } + // A merged PR can land for an issue that is already `done` — a duplicate + // webhook, or a concurrent path that won the race. The locked pre-image tells + // us whether THIS call actually transitioned it. On a no-op we already emitted + // no domain event; the parent notification and realtime status_changed must + // be suppressed on the SAME condition, else a no-op re-drives the child-done + // comment / trigger and a spurious "issue updated" (review point 4). + if before.Status == updated.Status { + return + } + // Fire the platform parent-notification path on the same transition the - // HTTP UpdateIssue / BatchUpdateIssues paths use. A merged PR is one of - // the most common ways a sub-issue actually reaches `done`, and skipping - // it here would leave the parent silent for the dominant completion path. - // notifyParentOfChildDone re-checks every guard (prev != done, parent - // exists, parent not terminal), so calling it unconditionally is safe. - h.notifyParentOfChildDone(ctx, issue, updated) + // HTTP UpdateIssue / BatchUpdateIssues paths use, driven by the locked + // pre-image. A merged PR is one of the most common ways a sub-issue actually + // reaches `done`. notifyParentOfChildDone re-checks every guard (prev != done, + // parent exists, parent not terminal). + h.notifyParentOfChildDone(ctx, before, updated) prefix := h.getIssuePrefix(ctx, issue.WorkspaceID) resp := issueToResponse(updated, prefix) h.publish(protocol.EventIssueUpdated, workspaceID, "system", "", map[string]any{ "issue": resp, "status_changed": true, - "prev_status": issue.Status, - "creator_type": issue.CreatorType, - "creator_id": uuidToString(issue.CreatorID), + "prev_status": before.Status, + "creator_type": before.CreatorType, + "creator_id": uuidToString(before.CreatorID), "source": "github_pr_merged", }) } diff --git a/server/internal/handler/handler.go b/server/internal/handler/handler.go index 33543a12eaa..612c667ba8c 100644 --- a/server/internal/handler/handler.go +++ b/server/internal/handler/handler.go @@ -139,6 +139,7 @@ type Handler struct { TaskService *service.TaskService IssueService *service.IssueService AutopilotService *service.AutopilotService + HookService *service.HookService EmailService *service.EmailService UpdateStore UpdateStore ModelListStore ModelListStore @@ -275,6 +276,7 @@ func New(queries *db.Queries, txStarter txStarter, hub *realtime.Hub, bus *event TaskService: taskSvc, IssueService: service.NewIssueService(queries, txStarter, bus, analyticsClient, taskSvc), AutopilotService: service.NewAutopilotService(queries, txStarter, bus, taskSvc), + HookService: service.NewHookService(queries, txStarter), EmailService: emailService, UpdateStore: NewInMemoryUpdateStore(), ModelListStore: NewInMemoryModelListStore(), diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go new file mode 100644 index 00000000000..46481963788 --- /dev/null +++ b/server/internal/handler/hook.go @@ -0,0 +1,415 @@ +package handler + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// hookExecutionListLimit bounds the execution-trace read for the debug endpoint. +const hookExecutionListLimit = 100 + +// HookResponse is the API view of a hook plus its active revision. +type HookResponse struct { + ID string `json:"id"` + WorkspaceID string `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Scope HookScopeResponse `json:"scope"` + Origin string `json:"origin"` + DisabledReason string `json:"disabled_reason,omitempty"` + Revision HookRevisionResponse `json:"revision"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// HookScopeResponse is the hook lifecycle scope. +type HookScopeResponse struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` +} + +// HookRevisionResponse is the active immutable revision. +type HookRevisionResponse struct { + ID string `json:"id"` + Revision int32 `json:"revision"` + Event string `json:"event"` + Match json.RawMessage `json:"match"` + Conditions json.RawMessage `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions json.RawMessage `json:"actions"` + CreatedAt string `json:"created_at"` +} + +// HookExecutionResponse is one row of the execution trace (all null in PR2 — no +// matcher runs yet — but shaped so the debug endpoint is stable for PR3). +type HookExecutionResponse struct { + ID string `json:"id"` + HookID string `json:"hook_id"` + RevisionID string `json:"hook_revision_id"` + EventID string `json:"event_id"` + Correlation string `json:"correlation_id"` + Status string `json:"status"` + SkipReason string `json:"skip_reason,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + CreatedAt string `json:"created_at"` +} + +func hookToResponse(hr service.HookWithRevision) HookResponse { + h, rev := hr.Hook, hr.Revision + return HookResponse{ + ID: uuidToString(h.ID), + WorkspaceID: uuidToString(h.WorkspaceID), + Name: h.Name, + Enabled: h.Enabled, + Scope: HookScopeResponse{Type: h.ScopeType, ID: uuidToString(h.ScopeID)}, + Origin: h.Origin, + DisabledReason: textValue(h.DisabledReason), + Revision: HookRevisionResponse{ + ID: uuidToString(rev.ID), + Revision: rev.Revision, + Event: rev.EventType, + Match: rawJSON(rev.Match), + Conditions: rawJSON(rev.Conditions), + FireMode: rev.FireMode, + Actions: rawJSON(rev.Actions), + CreatedAt: timestampToString(rev.CreatedAt), + }, + CreatedAt: timestampToString(h.CreatedAt), + UpdatedAt: timestampToString(h.UpdatedAt), + } +} + +func hookExecutionToResponse(e db.HookExecution) HookExecutionResponse { + return HookExecutionResponse{ + ID: uuidToString(e.ID), + HookID: uuidToString(e.HookID), + RevisionID: uuidToString(e.HookRevisionID), + EventID: uuidToString(e.EventID), + Correlation: uuidToString(e.CorrelationID), + Status: e.Status, + SkipReason: textValue(e.SkipReason), + ErrorCode: textValue(e.ErrorCode), + CreatedAt: timestampToString(e.CreatedAt), + } +} + +// hookEnabled gates every hook endpoint behind the automation_event_hooks flag. +// Store-only PR2 already writes rows, but the whole surface stays invisible until +// the flag is deliberately turned on, so nothing changes for existing workspaces. +func (h *Handler) hookEnabled(w http.ResponseWriter, r *http.Request) bool { + if !featureflags.EventHooksEnabled(r.Context(), h.FeatureFlags) { + writeError(w, http.StatusNotFound, "event hooks are not enabled") + return false + } + return true +} + +// CreateHook validates and persists a new hook + revision #1. +func (h *Handler) CreateHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return + } + + spec, ok := decodeHookSpec(w, r) + if !ok { + return + } + + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + + result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusCreated, hookToResponse(result)) +} + +// ListHooks returns every non-archived hook in the workspace. +func (h *Handler) ListHooks(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return + } + hooks, err := h.HookService.ListHooks(r.Context(), workspaceUUID) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]HookResponse, 0, len(hooks)) + for _, hook := range hooks { + resp = append(resp, hookToResponse(hook)) + } + writeJSON(w, http.StatusOK, resp) +} + +// GetHook returns one hook with its active revision. +func (h *Handler) GetHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + result, err := h.HookService.GetHook(r.Context(), workspaceUUID, hookUUID) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// UpdateHook appends a new revision and repoints the active pointer. +func (h *Handler) UpdateHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + spec, ok := decodeHookSpec(w, r) + if !ok { + return + } + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// EnableHook re-enables a disabled hook. +func (h *Handler) EnableHook(w http.ResponseWriter, r *http.Request) { + h.setHookEnabled(w, r, true) +} + +// DisableHook disables a hook with an optional reason. +func (h *Handler) DisableHook(w http.ResponseWriter, r *http.Request) { + h.setHookEnabled(w, r, false) +} + +func (h *Handler) setHookEnabled(w http.ResponseWriter, r *http.Request, enabled bool) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + reason := "" + if !enabled && r.ContentLength != 0 { + var body struct { + Reason string `json:"reason"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + reason = body.Reason + } + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + result, err := h.HookService.SetEnabled(r.Context(), workspaceUUID, hookUUID, enabled, reason, author) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// DeleteHook soft-archives a hook. +func (h *Handler) DeleteHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + if err := h.HookService.ArchiveHook(r.Context(), workspaceUUID, hookUUID, author); err != nil { + h.writeHookError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// ListHookExecutions returns the newest execution-trace rows for a hook. +func (h *Handler) ListHookExecutions(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + execs, err := h.HookService.ListExecutions(r.Context(), workspaceUUID, hookUUID, hookExecutionListLimit) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]HookExecutionResponse, 0, len(execs)) + for _, e := range execs { + resp = append(resp, hookExecutionToResponse(e)) + } + writeJSON(w, http.StatusOK, resp) +} + +// hookPathParams resolves the workspace and the {id} hook UUID for a scoped route. +func (h *Handler) hookPathParams(w http.ResponseWriter, r *http.Request) (pgtype.UUID, pgtype.UUID, bool) { + workspaceID := h.resolveWorkspaceID(r) + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return pgtype.UUID{}, pgtype.UUID{}, false + } + hookUUID, ok := parseUUIDOrBadRequest(w, chi.URLParam(r, "id"), "id") + if !ok { + return pgtype.UUID{}, pgtype.UUID{}, false + } + return workspaceUUID, hookUUID, true +} + +// decodeHookSpec strictly decodes the request body into a HookSpec, rejecting +// unknown fields so a stray top-level or nested key can never be silently +// accepted (MUL-4332 PR2 review point 3). Per-action disallowed fields are +// rejected by the automation validator. +func decodeHookSpec(w http.ResponseWriter, r *http.Request) (automation.HookSpec, bool) { + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + var spec automation.HookSpec + if err := dec.Decode(&spec); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error()) + return automation.HookSpec{}, false + } + // The body must be exactly one JSON value. A second Decode must hit io.EOF; + // anything else means trailing data (a smuggled second document) that + // DisallowUnknownFields alone does not catch (MUL-4332 review round 3, point 4). + if err := dec.Decode(new(json.RawMessage)); err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body: expected exactly one JSON document") + return automation.HookSpec{}, false + } + return spec, true +} + +// resolveHookWriter derives the audit creator actor, the accountable human +// principal (§8), and whether that principal is a workspace owner/admin, plus the +// agent-admission callback used for fail-closed trigger_agent validation. A +// member acts under their own authority; an agent must resolve to the human +// behind its current task. The resolved principal must still be a workspace +// member (review point 2) — an agent UUID is never a substitute for a human, and +// a departed principal can no longer author hooks. +func (h *Handler) resolveHookWriter(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, bool) { + actorType, actorID := h.resolveActor(r, userID, workspaceID) + actorUUID, err := util.ParseUUID(actorID) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid actor id") + return service.HookAuthor{}, false + } + principal := h.invokeOriginatorFromRequest(r, actorType, actorID) + if principal == "" { + h.writeDispatchBlocked(w, http.StatusForbidden, ReasonInvocationNotAllowed) + return service.HookAuthor{}, false + } + principalUUID, err := util.ParseUUID(principal) + if err != nil { + writeError(w, http.StatusForbidden, "no accountable authorization principal") + return service.HookAuthor{}, false + } + // Membership, role, and every target admission are (re)checked by the service + // inside the write transaction against this principal, so a stale snapshot can + // never authorize the write. + return service.HookAuthor{ActorType: actorType, ActorID: actorUUID, PrincipalUserID: principalUUID}, true +} + +func (h *Handler) writeHookError(w http.ResponseWriter, err error) { + if ve, ok := automation.AsValidationError(err); ok { + writeError(w, http.StatusBadRequest, ve.Error()) + return + } + switch { + case errors.Is(err, service.ErrHookNotFound): + writeError(w, http.StatusNotFound, "hook not found") + case errors.Is(err, service.ErrHookEventNotFound): + writeError(w, http.StatusNotFound, "event not found") + case errors.Is(err, service.ErrHookSystemManaged): + writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookForbidden): + writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookPrincipalDeparted): + writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookNoPrincipal): + writeError(w, http.StatusForbidden, "no accountable authorization principal") + default: + writeError(w, http.StatusInternalServerError, "hook request failed") + } +} + +// rawJSON returns b as json.RawMessage, defaulting an empty column to a JSON null. +func rawJSON(b []byte) json.RawMessage { + if len(b) == 0 { + return json.RawMessage("null") + } + return json.RawMessage(b) +} + +// textValue unwraps a nullable text column to a plain string ("" when NULL). +func textValue(t pgtype.Text) string { + if t.Valid { + return t.String + } + return "" +} diff --git a/server/internal/handler/hook_eval.go b/server/internal/handler/hook_eval.go new file mode 100644 index 00000000000..1d95f67c174 --- /dev/null +++ b/server/internal/handler/hook_eval.go @@ -0,0 +1,176 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/multica-ai/multica/server/internal/automation" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// hookEventListLimit is defensive; a correlation chain is bounded by the depth / +// width guardrails, but cap the debug read anyway. +const hookEventListLimit = 1000 + +// DryRunHookRequest dry-runs a candidate spec against a historical event. +type DryRunHookRequest struct { + Hook automation.HookSpec `json:"hook"` + EventID string `json:"event_id"` +} + +// ExplainHookRequest explains a stored hook revision's decision for an event. +type ExplainHookRequest struct { + HookID string `json:"hook_id"` + EventID string `json:"event_id"` + Revision int32 `json:"revision,omitempty"` +} + +// DomainEventResponse is the read-only API view of a domain event. Payload is +// projected to the event's declared schema fields (fail-closed redaction); the +// causation_* fields locate the direct action that produced the event. +type DomainEventResponse struct { + ID string `json:"id"` + Seq int64 `json:"seq"` + Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` + SubjectType string `json:"subject_type"` + SubjectID string `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID string `json:"actor_id,omitempty"` + Payload json.RawMessage `json:"payload"` + CorrelationID string `json:"correlation_id"` + CausationExecutionID string `json:"causation_execution_id,omitempty"` + CausationActionIndex *int32 `json:"causation_action_index,omitempty"` + HopCount int32 `json:"hop_count"` + CreatedAt string `json:"created_at"` +} + +// DryRunHook evaluates a candidate hook spec against a historical event without +// executing anything or mutating any durable state (§10, decision 2A). +func (h *Handler) DryRunHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + var req DryRunHookRequest + if !decodeJSONBodyStrict(w, r, &req) { + return + } + eventUUID, ok := parseUUIDOrBadRequest(w, req.EventID, "event_id") + if !ok { + return + } + result, err := h.HookService.DryRun(r.Context(), workspaceUUID, req.Hook, eventUUID) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// ExplainHook explains why a stored hook (its active or a specified revision) +// would or would not fire for a historical event. Read-only. +func (h *Handler) ExplainHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + var req ExplainHookRequest + if !decodeJSONBodyStrict(w, r, &req) { + return + } + hookUUID, ok := parseUUIDOrBadRequest(w, req.HookID, "hook_id") + if !ok { + return + } + eventUUID, ok := parseUUIDOrBadRequest(w, req.EventID, "event_id") + if !ok { + return + } + result, err := h.HookService.Explain(r.Context(), workspaceUUID, hookUUID, eventUUID, req.Revision) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// ListEventsByCorrelation returns the domain events in a correlation chain, for +// execution-chain debugging (GET /api/events?correlation_id=). Read-only. +func (h *Handler) ListEventsByCorrelation(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + correlationUUID, ok := parseUUIDOrBadRequest(w, r.URL.Query().Get("correlation_id"), "correlation_id") + if !ok { + return + } + events, err := h.HookService.EventsByCorrelation(r.Context(), workspaceUUID, correlationUUID, hookEventListLimit) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]DomainEventResponse, 0, len(events)) + for _, e := range events { + resp = append(resp, domainEventToResponse(e)) + } + writeJSON(w, http.StatusOK, resp) +} + +func domainEventToResponse(e db.DomainEvent) DomainEventResponse { + // Fail-closed schema projection: expose only the event type's declared + // payload fields; drop free-text / undeclared / sensitive keys. + var payload map[string]any + if len(e.Payload) > 0 { + _ = json.Unmarshal(e.Payload, &payload) + } + projected, _ := json.Marshal(automation.ProjectPayload(e.Type, payload)) + + resp := DomainEventResponse{ + ID: uuidToString(e.ID), + Seq: e.Seq, + Type: e.Type, + SchemaVersion: e.SchemaVersion, + SubjectType: e.SubjectType, + SubjectID: uuidToString(e.SubjectID), + ActorType: e.ActorType, + ActorID: uuidToString(e.ActorID), + Payload: json.RawMessage(projected), + CorrelationID: uuidToString(e.CorrelationID), + CausationExecutionID: uuidToString(e.CausationExecutionID), + HopCount: e.HopCount, + CreatedAt: timestampToString(e.CreatedAt), + } + if e.CausationActionIndex.Valid { + idx := e.CausationActionIndex.Int32 + resp.CausationActionIndex = &idx + } + return resp +} + +// decodeJSONBodyStrict decodes exactly one JSON document into dst, rejecting +// unknown fields and any trailing second document. +func decodeJSONBodyStrict(w http.ResponseWriter, r *http.Request, dst any) bool { + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error()) + return false + } + if err := dec.Decode(new(json.RawMessage)); err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body: expected exactly one JSON document") + return false + } + return true +} diff --git a/server/internal/handler/hook_eval_test.go b/server/internal/handler/hook_eval_test.go new file mode 100644 index 00000000000..5b4e16532bb --- /dev/null +++ b/server/internal/handler/hook_eval_test.go @@ -0,0 +1,314 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/multica-ai/multica/server/internal/automation" +) + +// seedStatusChangedEvent inserts an issue.status_changed domain_event whose +// subject is issueID, with a from/to payload and the given correlation id. +func seedStatusChangedEvent(t *testing.T, issueID, from, to, correlationID string) string { + t.Helper() + var id string + payload := fmt.Sprintf(`{"from":%q,"to":%q}`, from, to) + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id) + VALUES ($1, 'issue.status_changed', 1, 'issue', $2, 'member', $3, $4::jsonb, $5) + RETURNING id`, + testWorkspaceID, issueID, testUserID, payload, correlationID).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + return id +} + +// seedDomainEvent inserts a domain_event with an explicit workspace, type and +// raw JSON payload. +func seedDomainEvent(t *testing.T, workspaceID, typ, subjectID, payloadJSON, correlationID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id) + VALUES ($1, $2, 1, 'issue', $3, 'member', $4, $5::jsonb, $6) + RETURNING id`, + workspaceID, typ, subjectID, testUserID, payloadJSON, correlationID).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + return id +} + +func decodeEvaluation(t *testing.T, w *httptest.ResponseRecorder) automation.Evaluation { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var ev automation.Evaluation + if err := json.NewDecoder(w.Body).Decode(&ev); err != nil { + t.Fatalf("decode evaluation: %v", err) + } + return ev +} + +func TestHookDryRun(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + eventID := seedStatusChangedEvent(t, issueID, "in_progress", "done", issueID) + + // A spec matching to=done fires; evaluated against current state. + w := httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": sampleHookSpec("dr", "hi", issueID), "event_id": eventID})) + ev := decodeEvaluation(t, w) + if !ev.Matched || !ev.Eligible || ev.DecisionComplete || ev.Reason != automation.ReasonMatched { + t.Fatalf("expected match, got %+v", ev) + } + if ev.EvaluatedAgainst != automation.EvaluatedAgainstCurrentState { + t.Errorf("evaluated_against = %q, want current_state", ev.EvaluatedAgainst) + } + + // A spec whose match requires to=blocked does not fire. + spec := sampleHookSpec("dr2", "hi", issueID) + spec["when"].(map[string]any)["match"] = map[string]any{"to": "blocked"} + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + ev = decodeEvaluation(t, w) + if ev.Matched || ev.Reason != automation.ReasonNoMatch { + t.Errorf("expected no_match, got %+v", ev) + } + + // A missing event is 404. + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": sampleHookSpec("dr3", "hi", issueID), "event_id": "99999999-9999-9999-9999-999999999999"})) + if w.Code != http.StatusNotFound { + t.Errorf("dry-run missing event: status %d, want 404", w.Code) + } + + // An invalid spec is 400 (shape validation), never reaching evaluation. + bad := sampleHookSpec("bad", "hi", issueID) + bad["do"] = []any{map[string]any{"type": "set_issue_status_many"}} + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": bad, "event_id": eventID})) + if w.Code != http.StatusBadRequest { + t.Errorf("dry-run invalid spec: status %d, want 400", w.Code) + } +} + +// dry-run reads conditions against CURRENT workspace state, not the event moment. +func TestHookDryRunConditionsUseCurrentState(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) // seeded as status 'todo' + eventID := seedStatusChangedEvent(t, issueID, "todo", "in_progress", issueID) + + spec := sampleHookSpec("cond", "hi", issueID) + spec["when"].(map[string]any)["match"] = map[string]any{} + spec["if"] = []any{map[string]any{"issues_status": map[string]any{"ids": []any{issueID}, "all": "done"}}} + + // Issue is currently 'todo' → condition (all done) is false. + w := httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + if ev := decodeEvaluation(t, w); ev.ConditionsMet || ev.Reason != automation.ReasonConditionFalse { + t.Fatalf("expected condition_false with issue todo, got %+v", ev) + } + + // Move the issue to done → the same dry-run now reports conditions met. + if _, err := testPool.Exec(ctx, `UPDATE issue SET status = 'done' WHERE id = $1`, issueID); err != nil { + t.Fatalf("update issue: %v", err) + } + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + if ev := decodeEvaluation(t, w); !ev.ConditionsMet || ev.Reason != automation.ReasonMatched { + t.Fatalf("expected conditions met after issue moved to done, got %+v", ev) + } +} + +func TestHookExplain(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + hook := createHookAs(t, testUserID, sampleHookSpec("explain hook", "hi", issueID)) // matches to=done + + matchEvent := seedStatusChangedEvent(t, issueID, "in_progress", "done", issueID) + noMatchEvent := seedStatusChangedEvent(t, issueID, "in_progress", "todo", issueID) + + w := httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": hook.ID, "event_id": matchEvent})) + if ev := decodeEvaluation(t, w); ev.Reason != automation.ReasonMatched { + t.Errorf("explain matching event: reason %q, want matched (%+v)", ev.Reason, ev) + } + + w = httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": hook.ID, "event_id": noMatchEvent})) + if ev := decodeEvaluation(t, w); ev.Reason != automation.ReasonNoMatch { + t.Errorf("explain non-matching event: reason %q, want no_match", ev.Reason) + } + + // Unknown hook is 404. + w = httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": "88888888-8888-8888-8888-888888888888", "event_id": matchEvent})) + if w.Code != http.StatusNotFound { + t.Errorf("explain unknown hook: status %d, want 404", w.Code) + } +} + +func TestHookEventsByCorrelation(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + seedStatusChangedEvent(t, issueID, "todo", "in_progress", correlation) + seedStatusChangedEvent(t, issueID, "in_progress", "done", correlation) + // A different correlation must not leak in. + seedStatusChangedEvent(t, issueID, "todo", "done", "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 2 { + t.Fatalf("events = %d, want 2 (the correlation chain only)", len(events)) + } + for _, e := range events { + if e.CorrelationID != correlation { + t.Errorf("leaked event from correlation %s", e.CorrelationID) + } + if e.SchemaVersion != 1 { + t.Errorf("schema_version = %d, want 1", e.SchemaVersion) + } + } +} + +// The correlation payload is projected to the event schema: free-text / undeclared +// keys (e.g. an issue title) are redacted, declared fields survive (review point 3). +func TestHookCorrelationRedactsPayload(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "cccccccc-cccc-cccc-cccc-cccccccccccc" + seedDomainEvent(t, testWorkspaceID, "issue.created", issueID, + `{"status":"todo","priority":"high","title":"SECRET internal title"}`, correlation) + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 1 { + t.Fatalf("events = %d, want 1", len(events)) + } + var payload map[string]any + json.Unmarshal(events[0].Payload, &payload) + if _, leaked := payload["title"]; leaked { + t.Errorf("free-text title was not redacted: %v", payload) + } + if payload["status"] != "todo" || payload["priority"] != "high" { + t.Errorf("declared payload fields were dropped: %v", payload) + } +} + +// A same-correlation event in a DIFFERENT workspace must never appear. +func TestHookCorrelationCrossWorkspaceFiltered(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "dddddddd-dddd-dddd-dddd-dddddddddddd" + seedDomainEvent(t, testWorkspaceID, "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + // domain_event has no FK on workspace_id, so an arbitrary foreign workspace id + // with the SAME correlation must be filtered out by the workspace-scoped query. + seedDomainEvent(t, "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 1 { + t.Fatalf("events = %d, want 1 (foreign-workspace same-correlation must be filtered)", len(events)) + } +} + +// The chain limit is enforced in the query, not by truncating a fully-loaded chain. +func TestHookCorrelationLimitPushedToQuery(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + issueID := seedHookIssue(t) + const correlation = "ffffffff-ffff-ffff-ffff-ffffffffffff" + for i := 0; i < 3; i++ { + seedDomainEvent(t, testWorkspaceID, "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + } + got, err := testHandler.HookService.EventsByCorrelation(ctx, parseUUID(testWorkspaceID), parseUUID(correlation), 2) + if err != nil { + t.Fatalf("events by correlation: %v", err) + } + if len(got) != 2 { + t.Errorf("query returned %d rows, want 2 (LIMIT must be applied in SQL)", len(got)) + } +} + +// The whole read-only surface is invisible unless the feature flag is on. +func TestHookEvalRequiresFeatureFlag(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + prev := testHandler.FeatureFlags + testHandler.FeatureFlags = nil + t.Cleanup(func() { testHandler.FeatureFlags = prev }) + + for _, tc := range []struct { + name string + call func(w *httptest.ResponseRecorder) + }{ + {"dry-run", func(w *httptest.ResponseRecorder) { + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", map[string]any{})) + }}, + {"explain", func(w *httptest.ResponseRecorder) { + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", map[string]any{})) + }}, + {"events", func(w *httptest.ResponseRecorder) { + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events", nil)) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + w := httptest.NewRecorder() + tc.call(w) + if w.Code != http.StatusNotFound { + t.Errorf("%s with flag off: status %d, want 404", tc.name, w.Code) + } + }) + } +} diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go new file mode 100644 index 00000000000..78720e41d1a --- /dev/null +++ b/server/internal/handler/hook_test.go @@ -0,0 +1,929 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// hookSpecFromMap round-trips a test spec map through JSON into a typed HookSpec, +// for tests that drive the service layer directly. +func hookSpecFromMap(m map[string]any) automation.HookSpec { + buf, _ := json.Marshal(m) + var spec automation.HookSpec + json.Unmarshal(buf, &spec) + return spec +} + +// enableHooksFlag flips automation_event_hooks on for the shared test handler and +// restores the previous flag service when the test ends. +func enableHooksFlag(t *testing.T) { + t.Helper() + prev := testHandler.FeatureFlags + p := featureflag.NewStaticProvider() + p.Set(featureflags.EventHooks, featureflag.Rule{Default: true}) + testHandler.FeatureFlags = featureflag.NewService(p) + t.Cleanup(func() { testHandler.FeatureFlags = prev }) +} + +func newMemberHookRequest(method, path string, body any) *http.Request { + return newUserHookRequest(method, path, body, testUserID) +} + +// newUserHookRequest builds a member-authenticated request for the given user. +func newUserHookRequest(method, path string, body any, userID string) *http.Request { + req := newJSONRequest(method, path, body) + req.Header.Set("X-User-ID", userID) + req.Header.Set("X-Workspace-ID", testWorkspaceID) + return req +} + +func newJSONRequest(method, path string, body any) *http.Request { + var r *http.Request + if body == nil { + r = httptest.NewRequest(method, path, nil) + } else { + buf, _ := json.Marshal(body) + r = httptest.NewRequest(method, path, bytes.NewReader(buf)) + } + r.Header.Set("Content-Type", "application/json") + return r +} + +// seedHookIssue inserts a real issue in the test workspace and returns its id. +func seedHookIssue(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO issue (workspace_id, title, status, priority, creator_type, creator_id, number) + VALUES ($1, 'hook target issue', 'todo', 'medium', 'member', $2, + COALESCE((SELECT MAX(number) FROM issue WHERE workspace_id = $1), 0) + 1) + RETURNING id`, testWorkspaceID, testUserID).Scan(&id); err != nil { + t.Fatalf("seed issue: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM issue WHERE id = $1`, id) }) + return id +} + +// seededHookAgentID returns the workspace-visible agent seeded by the fixture. +func seededHookAgentID(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), + `SELECT id FROM agent WHERE workspace_id = $1 AND name = 'Handler Test Agent' LIMIT 1`, + testWorkspaceID).Scan(&id); err != nil { + t.Fatalf("load seeded agent: %v", err) + } + return id +} + +// testOwnerMemberID returns the member row id of the fixture owner (testUserID). +func testOwnerMemberID(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), + `SELECT id FROM member WHERE workspace_id = $1 AND user_id = $2`, + testWorkspaceID, testUserID).Scan(&id); err != nil { + t.Fatalf("load owner member: %v", err) + } + return id +} + +// hookSeedCounter makes each seeded user's email unique across calls. +var hookSeedCounter atomic.Int64 + +// seedHookMember inserts a user + workspace member with the given role and +// returns the user id. +func seedHookMember(t *testing.T, role string) string { + t.Helper() + ctx := context.Background() + var userID string + email := fmt.Sprintf("hook-%s-%d-%d@test.local", role, hookSeedCounter.Add(1), len(t.Name())) + if err := testPool.QueryRow(ctx, + `INSERT INTO "user" (name, email) VALUES ($1, $2) RETURNING id`, + "Hook "+role, email).Scan(&userID); err != nil { + t.Fatalf("seed user: %v", err) + } + if _, err := testPool.Exec(ctx, + `INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, $3)`, + testWorkspaceID, userID, role); err != nil { + t.Fatalf("seed member: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, testWorkspaceID, userID) + testPool.Exec(context.Background(), `DELETE FROM "user" WHERE id = $1`, userID) + }) + return userID +} + +// sampleHookSpec is a minimal valid per_event spec: comment an existing issue. +func sampleHookSpec(name, message, issueID string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{ + "event": "issue.status_changed", + "match": map[string]any{"to": "done"}, + }, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{ + map[string]any{"type": "add_comment", "issue_id": issueID, "message": message}, + }, + } +} + +func createHookAs(t *testing.T, userID string, spec map[string]any) HookResponse { + t.Helper() + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", spec, userID)) + if w.Code != http.StatusCreated { + t.Fatalf("create hook as %s: status %d: %s", userID, w.Code, w.Body.String()) + } + var resp HookResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode create: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM hook_revision WHERE hook_id = $1`, resp.ID) + testPool.Exec(context.Background(), `DELETE FROM hook WHERE id = $1`, resp.ID) + }) + return resp +} + +func TestHookCRUDLifecycle(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + + created := createHookAs(t, testUserID, sampleHookSpec("lifecycle hook", "first", issueID)) + if created.Revision.Revision != 1 || !created.Enabled || created.Revision.Event != "issue.status_changed" { + t.Fatalf("unexpected create response: %+v", created) + } + hookID := created.ID + + // Get. + w := httptest.NewRecorder() + testHandler.GetHook(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("get: status %d: %s", w.Code, w.Body.String()) + } + + // Update → new immutable revision #2, active pointer moves, name updated. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hookID, sampleHookSpec("renamed hook", "second", issueID)), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("update: status %d: %s", w.Code, w.Body.String()) + } + var updated HookResponse + json.NewDecoder(w.Body).Decode(&updated) + if updated.Revision.Revision != 2 || updated.Name != "renamed hook" || updated.Revision.ID == created.Revision.ID { + t.Fatalf("update did not append a new revision / rename: %+v", updated) + } + var revCount int + testPool.QueryRow(ctx, `SELECT count(*) FROM hook_revision WHERE hook_id = $1`, hookID).Scan(&revCount) + if revCount != 2 { + t.Errorf("hook_revision count = %d, want 2 (revisions are immutable)", revCount) + } + + // List. + w = httptest.NewRecorder() + testHandler.ListHooks(w, newMemberHookRequest(http.MethodGet, "/api/hooks", nil)) + var list []HookResponse + json.NewDecoder(w.Body).Decode(&list) + if !containsHook(list, hookID) { + t.Errorf("list does not contain created hook %s", hookID) + } + + // Disable → enabled=false with reason. + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hookID+"/disable", map[string]any{"reason": "paused"}), "id", hookID)) + var disabled HookResponse + json.NewDecoder(w.Body).Decode(&disabled) + if disabled.Enabled || disabled.DisabledReason != "paused" { + t.Errorf("disable did not take: %+v", disabled) + } + + // Enable again. + w = httptest.NewRecorder() + testHandler.EnableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hookID+"/enable", nil), "id", hookID)) + var reenabled HookResponse + json.NewDecoder(w.Body).Decode(&reenabled) + if !reenabled.Enabled { + t.Errorf("enable did not take: %+v", reenabled) + } + + // Executions trace is empty in store-only PR2 but the endpoint works. + w = httptest.NewRecorder() + testHandler.ListHookExecutions(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID+"/executions", nil), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("executions: status %d", w.Code) + } + + // Delete (soft archive) → subsequent get 404s. + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newMemberHookRequest(http.MethodDelete, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusNoContent { + t.Fatalf("delete: status %d: %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.GetHook(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusNotFound { + t.Errorf("get after archive: status %d, want 404", w.Code) + } +} + +func TestHookRequiresFeatureFlag(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + prev := testHandler.FeatureFlags + testHandler.FeatureFlags = nil // nil service → every flag resolves to its default (off) + t.Cleanup(func() { testHandler.FeatureFlags = prev }) + + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("blocked", "x", "55555555-5555-5555-5555-555555555555"))) + if w.Code != http.StatusNotFound { + t.Errorf("create with flag off: status %d, want 404", w.Code) + } +} + +// Strict schema: unknown fields and per-action disallowed fields and bad status +// are all rejected at the boundary with 400, never persisted (review point 3). +func TestHookStrictSchemaRejections(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + + cases := map[string]func() map[string]any{ + "unknown top-level field": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["unexpected"] = true + return s + }, + "disallowed action field": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "add_comment", "issue_id": issueID, "message": "hi", "agent_id": "66666666-6666-6666-6666-666666666666"}} + return s + }, + "invalid status enum": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "set_issue_status", "issue_id": issueID, "status": "ascended"}} + return s + }, + "system-only action": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "set_issue_status_many"}} + return s + }, + } + for name, build := range cases { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", build())) + if w.Code != http.StatusBadRequest { + t.Errorf("status %d, want 400: %s", w.Code, w.Body.String()) + } + }) + } +} + +// Fail-closed target validation: a spec that references a target absent from the +// workspace is rejected with 400, not persisted (review point 2). +func TestHookFailClosedTargets(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + realIssue := seedHookIssue(t) + const ghost = "77777777-7777-7777-7777-777777777777" + + action := func(a map[string]any) map[string]any { + s := sampleHookSpec("targets", "hi", realIssue) + s["do"] = []any{a} + return s + } + cases := map[string]map[string]any{ + "nonexistent issue": action(map[string]any{"type": "add_comment", "issue_id": ghost, "message": "hi"}), + "nonexistent member": action(map[string]any{"type": "send_inbox", "member_id": ghost, "message": "hi"}), + "nonexistent agent": action(map[string]any{"type": "trigger_agent", "issue_id": realIssue, "agent_id": ghost}), + "nonexistent autopilot": action(map[string]any{"type": "run_autopilot", "autopilot_id": ghost}), + } + for name, spec := range cases { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", spec)) + if w.Code != http.StatusBadRequest { + t.Errorf("status %d, want 400 (fail-closed): %s", w.Code, w.Body.String()) + } + }) + } + + // A real, invokable agent target is accepted. + t.Run("real invokable agent accepted", func(t *testing.T) { + spec := action(map[string]any{"type": "trigger_agent", "issue_id": realIssue, "agent_id": seededHookAgentID(t)}) + createHookAs(t, testUserID, spec) // fails the test if not 201 + }) +} + +// Only the hook's principal or a workspace owner/admin may edit it; an arbitrary +// member cannot rewrite a rule that keeps running under someone else's authority +// (review point 1). +func TestHookEditAuthorization(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + + author := seedHookMember(t, "member") // principal, non-admin + other := seedHookMember(t, "member") // non-principal, non-admin + + hook := createHookAs(t, author, sampleHookSpec("owned by author", "hi", issueID)) + + // A different non-admin member cannot edit / disable / delete it. + patch := withURLParam(newUserHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("hijacked", "x", issueID), other), "id", hook.ID) + w := httptest.NewRecorder() + testHandler.UpdateHook(w, patch) + if w.Code != http.StatusForbidden { + t.Errorf("other member PATCH: status %d, want 403", w.Code) + } + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newUserHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/disable", nil, other), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("other member disable: status %d, want 403", w.Code) + } + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newUserHookRequest(http.MethodDelete, "/api/hooks/"+hook.ID, nil, other), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("other member delete: status %d, want 403", w.Code) + } + + // The principal can edit their own hook. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newUserHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("by principal", "x", issueID), author), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("principal PATCH: status %d, want 200: %s", w.Code, w.Body.String()) + } + + // A workspace owner/admin (the fixture owner) can edit any hook. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("by admin", "x", issueID)), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("admin PATCH: status %d, want 200: %s", w.Code, w.Body.String()) + } + + // The principal was NOT transferred by any edit — it is still the author. + var principal string + testPool.QueryRow(context.Background(), + `SELECT authorization_principal_user_id FROM hook WHERE id = $1`, hook.ID).Scan(&principal) + if principal != author { + t.Errorf("principal = %s, want %s (edits must not transfer the principal)", principal, author) + } +} + +// Concurrent PATCHes on the same hook must each append a distinct, contiguous +// revision without a MAX+1 unique-index collision surfacing as a 500 (review +// point 4). Driven at the service layer so all workers hit the pool concurrently. +func TestHookConcurrentPatchAppendsContiguousRevisions(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + hook := createHookAs(t, testUserID, sampleHookSpec("concurrent", "hi", issueID)) + + admin := service.HookAuthor{ + ActorType: "member", + ActorID: parseUUID(testUserID), + PrincipalUserID: parseUUID(testUserID), + } + hookUUID := parseUUID(hook.ID) + wsUUID := parseUUID(testWorkspaceID) + + const workers = 8 + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + <-start + spec := hookSpecFromMap(sampleHookSpec(fmt.Sprintf("rev-%d", n), "x", issueID)) + _, err := testHandler.HookService.UpdateHook(ctx, wsUUID, hookUUID, spec, admin) + errs <- err + }(i) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent PATCH errored (revision race not serialized): %v", err) + } + } + + // Revisions must be exactly 1..(workers+1), contiguous and unique. + rows, err := testPool.Query(ctx, `SELECT revision FROM hook_revision WHERE hook_id = $1 ORDER BY revision`, hook.ID) + if err != nil { + t.Fatalf("query revisions: %v", err) + } + defer rows.Close() + var revs []int + for rows.Next() { + var r int + rows.Scan(&r) + revs = append(revs, r) + } + if len(revs) != workers+1 { + t.Fatalf("revision count = %d, want %d", len(revs), workers+1) + } + for i, r := range revs { + if r != i+1 { + t.Fatalf("revisions not contiguous at index %d: got %d, want %d (revs=%v)", i, r, i+1, revs) + } + } +} + +// seedPrivateAgent inserts a private agent owned by ownerUserID. +func seedPrivateAgent(t *testing.T, ownerUserID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO agent (workspace_id, name, description, runtime_mode, runtime_config, + runtime_id, visibility, permission_mode, max_concurrent_tasks, owner_id) + VALUES ($1, $2, '', 'cloud', '{}'::jsonb, $3, 'private', 'private', 1, $4) + RETURNING id`, + testWorkspaceID, fmt.Sprintf("Private Agent %d", hookSeedCounter.Add(1)), testRuntimeID, ownerUserID).Scan(&id); err != nil { + t.Fatalf("seed private agent: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM agent WHERE id = $1`, id) }) + return id +} + +// seedAutopilot inserts an autopilot created by creatorUserID assigned to the +// seeded (workspace-visible) agent. +func seedAutopilot(t *testing.T, creatorUserID string) string { + return seedAutopilotWith(t, creatorUserID, "agent", seededHookAgentID(t)) +} + +// seedAutopilotWith inserts an autopilot with an explicit assignee. +func seedAutopilotWith(t *testing.T, creatorUserID, assigneeType, assigneeID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO autopilot (workspace_id, title, assignee_type, assignee_id, status, execution_mode, created_by_type, created_by_id) + VALUES ($1, $2, $3, $4, 'active', 'run_only', 'member', $5) + RETURNING id`, + testWorkspaceID, "hook autopilot", assigneeType, assigneeID, creatorUserID).Scan(&id); err != nil { + t.Fatalf("seed autopilot: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM autopilot WHERE id = $1`, id) }) + return id +} + +func triggerAgentSpec(name, issueID, agentID string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{"event": "issue.status_changed"}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "trigger_agent", "issue_id": issueID, "agent_id": agentID}}, + } +} + +func runAutopilotSpec(name, autopilotID string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{"event": "issue.status_changed"}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "run_autopilot", "autopilot_id": autopilotID}}, + } +} + +// Target admission is judged against the hook's STORED principal, not the editor. +// An admin editing member A's hook cannot smuggle in a trigger_agent that A could +// not invoke (review round 3, point 1). +func TestHookAdmissionUsesStoredPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + privateAgent := seedPrivateAgent(t, testUserID) // owned by the fixture owner + memberA := seedHookMember(t, "member") + + hook := createHookAs(t, memberA, sampleHookSpec("owned by A", "hi", issueID)) + + // Owner (admin) PATCHes A's hook to trigger the owner's private agent. Because + // admission is judged for A (the stored principal), who cannot invoke that + // private agent, the edit is rejected — the admin cannot expand A's reach. + w := httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, triggerAgentSpec("hijack", issueID, privateAgent)), "id", hook.ID)) + if w.Code != http.StatusBadRequest { + t.Errorf("admin PATCH adding a private agent A cannot invoke: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The owner's OWN hook may reference the owner's private agent (owner invokes it). + createHookAs(t, testUserID, triggerAgentSpec("owner hook", issueID, privateAgent)) +} + +// run_autopilot targets are gated by the stored principal's write permission on +// the autopilot, not mere existence (review round 3, point 2). +func TestHookRunAutopilotChecksWritePermission(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + memberA := seedHookMember(t, "member") + autopilotID := seedAutopilot(t, testUserID) // created by the owner + + // Member A has no write access to the owner's autopilot → rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", runAutopilotSpec("by A", autopilotID), memberA)) + if w.Code != http.StatusBadRequest { + t.Errorf("member A run_autopilot on autopilot they cannot write: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The autopilot's creator (owner) may reference it. + createHookAs(t, testUserID, runAutopilotSpec("by owner", autopilotID)) +} + +// issue_field operand ids must resolve to workspace resources, not just the +// subject issue (review round 3, point 2). +func TestHookIssueFieldOperandValidation(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + realIssue := seedHookIssue(t) + const ghost = "77777777-7777-7777-7777-777777777777" + + parentCond := func(operand string) map[string]any { + return map[string]any{ + "name": "operand", + "when": map[string]any{"event": "issue.status_changed"}, + "if": []any{map[string]any{"issue_field": map[string]any{"id": realIssue, "field": "parent_issue_id", "eq": operand}}}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "add_comment", "issue_id": realIssue, "message": "hi"}}, + } + } + // A ghost parent operand is rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", parentCond(ghost))) + if w.Code != http.StatusBadRequest { + t.Errorf("issue_field parent operand ghost: status %d, want 400: %s", w.Code, w.Body.String()) + } + // A real parent operand is accepted. + createHookAs(t, testUserID, parentCond(realIssue)) +} + +// The write transaction re-checks that the principal is a workspace member; a +// valid-but-non-member principal cannot persist a hook (review round 3, point 3). +func TestHookServiceRejectsNonMemberPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + issueID := seedHookIssue(t) + author := service.HookAuthor{ + ActorType: "member", + ActorID: parseUUID("88888888-8888-8888-8888-888888888888"), + PrincipalUserID: parseUUID("88888888-8888-8888-8888-888888888888"), // not a member + } + _, err := testHandler.HookService.CreateHook(ctx, parseUUID(testWorkspaceID), hookSpecFromMap(sampleHookSpec("ghost", "hi", issueID)), author) + if err == nil { + t.Fatalf("expected rejection for non-member principal, got nil") + } + var count int + testPool.QueryRow(ctx, `SELECT count(*) FROM hook WHERE authorization_principal_user_id = $1`, "88888888-8888-8888-8888-888888888888").Scan(&count) + if count != 0 { + t.Errorf("a hook was persisted for a non-member principal (count=%d)", count) + } +} + +// The body must be exactly one JSON document; a smuggled trailing document is +// rejected even though DisallowUnknownFields alone would accept it (review round +// 3, point 4). +func TestHookRejectsTrailingJSONDocument(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + first, _ := json.Marshal(sampleHookSpec("first", "hi", issueID)) + body := append(append([]byte{}, first...), []byte(` {"unexpected":"second document"}`)...) + req := httptest.NewRequest(http.MethodPost, "/api/hooks", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-User-ID", testUserID) + req.Header.Set("X-Workspace-ID", testWorkspaceID) + w := httptest.NewRecorder() + testHandler.CreateHook(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("trailing second JSON document: status %d, want 400: %s", w.Code, w.Body.String()) + } +} + +// A hook's stored principal must remain a live member for the hook to be +// re-armed. After the principal leaves the workspace, update and enable are +// blocked (403), but disable/archive stay allowed for admins as safe degrading +// operations (review round 4, point 1). +func TestHookDepartedPrincipalGate(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + + // A leaves the workspace. + if _, err := testPool.Exec(ctx, `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, testWorkspaceID, memberA); err != nil { + t.Fatalf("remove member A: %v", err) + } + + // Owner (admin) may no longer PATCH or enable A's hook (re-arming). + w := httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("x", "y", issueID)), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("PATCH after principal departed: status %d, want 403: %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.EnableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/enable", nil), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("enable after principal departed: status %d, want 403: %s", w.Code, w.Body.String()) + } + + // Disable and archive are degrading ops an admin may still perform. + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/disable", nil), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("disable after principal departed: status %d, want 200 (degrading): %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newMemberHookRequest(http.MethodDelete, "/api/hooks/"+hook.ID, nil), "id", hook.ID)) + if w.Code != http.StatusNoContent { + t.Errorf("archive after principal departed: status %d, want 204 (degrading): %s", w.Code, w.Body.String()) + } +} + +// run_autopilot admission also gates the autopilot's assignee: a principal with +// write access (collaborator) whose autopilot targets an agent they cannot invoke +// is rejected at save (review round 4, point 2). +func TestHookAutopilotAssigneeAdmission(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + privateAgent := seedPrivateAgent(t, testUserID) // owner's private agent + autopilotID := seedAutopilotWith(t, testUserID, "agent", privateAgent) + memberA := seedHookMember(t, "member") + + // A is granted collaborator write access on the autopilot... + if _, err := testPool.Exec(ctx, ` + INSERT INTO autopilot_collaborator (autopilot_id, user_type, user_id, granted_by) + VALUES ($1, 'member', $2, $3)`, autopilotID, memberA, testUserID); err != nil { + t.Fatalf("grant collaborator: %v", err) + } + // autopilot_collaborator has no FK/cascade, so drop the grant explicitly to keep + // the shared test DB clean across repeat runs. + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM autopilot_collaborator WHERE autopilot_id = $1`, autopilotID) + }) + + // ...but A cannot invoke the autopilot's private-agent assignee, so save fails. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", runAutopilotSpec("by A collab", autopilotID), memberA)) + if w.Code != http.StatusBadRequest { + t.Errorf("collaborator run_autopilot with un-invocable assignee: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The owner owns the assignee agent, so their own hook is accepted. + createHookAs(t, testUserID, runAutopilotSpec("by owner", autopilotID)) +} + +// issue_field.assignee_id operands are polymorphic: a member operand is a USER id +// (previously wrongly rejected), plus agent and squad (review round 4, point 3). +func TestHookAssigneeOperandTypes(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + realIssue := seedHookIssue(t) + agentID := seededHookAgentID(t) + squadID := seedSquad(t, agentID, fmt.Sprintf("Hook Squad %d", hookSeedCounter.Add(1)), false) + const ghost = "77777777-7777-7777-7777-777777777777" + + assigneeCond := func(operand string) map[string]any { + return map[string]any{ + "name": "assignee", + "when": map[string]any{"event": "issue.status_changed"}, + "if": []any{map[string]any{"issue_field": map[string]any{"id": realIssue, "field": "assignee_id", "eq": operand}}}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "add_comment", "issue_id": realIssue, "message": "hi"}}, + } + } + // member USER id, agent id, squad id are all accepted. + createHookAs(t, testUserID, assigneeCond(testUserID)) // member operand is a user id + createHookAs(t, testUserID, assigneeCond(agentID)) + createHookAs(t, testUserID, assigneeCond(squadID)) + + // A ghost assignee operand is rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", assigneeCond(ghost))) + if w.Code != http.StatusBadRequest { + t.Errorf("ghost assignee operand: status %d, want 400: %s", w.Code, w.Body.String()) + } +} + +// runBlockedByMemberMutation opens an uncommitted member-row mutation (removal or +// role demotion), runs a hook op in a goroutine that is expected to BLOCK on the +// FOR SHARE membership read, asserts it blocks, then commits the mutation and +// returns the op's error. This deterministically reproduces the round-5 concurrency +// gap: the mutation lands before the hook write can proceed. +func runBlockedByMemberMutation(t *testing.T, mutationSQL string, args []any, op func() error) error { + t.Helper() + ctx := context.Background() + conn, err := testPool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire conn: %v", err) + } + defer conn.Release() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatalf("begin mutation tx: %v", err) + } + committed := false + defer func() { + if !committed { + tx.Rollback(ctx) + } + }() + if _, err := tx.Exec(ctx, mutationSQL, args...); err != nil { + t.Fatalf("member mutation: %v", err) + } + + done := make(chan error, 1) + go func() { done <- op() }() + + // The op must block on the locked member row while the mutation is uncommitted. + select { + case err := <-done: + t.Fatalf("hook op completed (err=%v) before the concurrent member mutation committed — FOR SHARE did not serialize", err) + case <-time.After(750 * time.Millisecond): + } + + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit member mutation: %v", err) + } + committed = true + return <-done +} + +func hookRevisionCount(t *testing.T, hookID string) int { + t.Helper() + var n int + if err := testPool.QueryRow(context.Background(), `SELECT count(*) FROM hook_revision WHERE hook_id = $1`, hookID).Scan(&n); err != nil { + t.Fatalf("count revisions: %v", err) + } + return n +} + +func memberAuthor(userID string) service.HookAuthor { + return service.HookAuthor{ActorType: "member", ActorID: parseUUID(userID), PrincipalUserID: parseUUID(userID)} +} + +// A concurrent removal of the hook's stored principal, committed while an update is +// mid-transaction, must make the update fail closed — not commit a new revision +// under the departed principal's authority (review round 5). +func TestHookUpdateBlocksOnConcurrentPrincipalRemoval(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + spec := hookSpecFromMap(sampleHookSpec("edited", "x", issueID)) + + err := runBlockedByMemberMutation(t, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, memberA}, + func() error { + // Owner (admin) edits A's hook; admission runs against the stored + // principal A, whose membership is being removed concurrently. + _, e := testHandler.HookService.UpdateHook(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), spec, memberAuthor(testUserID)) + return e + }) + if err == nil { + t.Fatalf("UpdateHook committed after the stored principal was removed mid-transaction") + } + if n := hookRevisionCount(t, hook.ID); n != 1 { + t.Errorf("hook_revision count = %d, want 1 (no revision may commit under a removed principal)", n) + } +} + +// Enable is a re-arming op, so a concurrent removal of the stored principal must +// also block it fail-closed. +func TestHookEnableBlocksOnConcurrentPrincipalRemoval(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + + // Disable it first (allowed) so a failed re-enable is observable. + if _, err := testHandler.HookService.SetEnabled(ctx, parseUUID(testWorkspaceID), parseUUID(hook.ID), false, "", memberAuthor(testUserID)); err != nil { + t.Fatalf("disable: %v", err) + } + + err := runBlockedByMemberMutation(t, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, memberA}, + func() error { + _, e := testHandler.HookService.SetEnabled(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), true, "", memberAuthor(testUserID)) + return e + }) + if err == nil { + t.Fatalf("EnableHook committed after the stored principal was removed mid-transaction") + } + var enabled bool + testPool.QueryRow(ctx, `SELECT enabled FROM hook WHERE id = $1`, hook.ID).Scan(&enabled) + if enabled { + t.Errorf("hook was re-enabled despite the principal being removed") + } +} + +// A concurrent admin→member demotion of the EDITOR, committed mid-transaction, must +// make the update fail closed (the editor is no longer authorized). +func TestHookUpdateBlocksOnConcurrentEditorDemotion(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") // stored principal (stays a member) + adminB := seedHookMember(t, "admin") // editor, demoted mid-transaction + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + spec := hookSpecFromMap(sampleHookSpec("edited", "x", issueID)) + + err := runBlockedByMemberMutation(t, + `UPDATE member SET role = 'member' WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, adminB}, + func() error { + _, e := testHandler.HookService.UpdateHook(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), spec, memberAuthor(adminB)) + return e + }) + if err == nil { + t.Fatalf("UpdateHook committed after the editor was demoted from admin mid-transaction") + } + if n := hookRevisionCount(t, hook.ID); n != 1 { + t.Errorf("hook_revision count = %d, want 1 (no revision may commit under a demoted editor)", n) + } +} + +// An agent author with no resolvable human principal (§8) is refused. +func TestHookAgentRequiresPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + req := newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("agent hook", "x", issueID)) + // Trusted agent identity (task_token), valid agent uuid, but no X-Task-ID means + // no originator can be resolved → no accountable principal. + req.Header.Set("X-Actor-Source", "task_token") + req.Header.Set("X-Agent-ID", "66666666-6666-6666-6666-666666666666") + w := httptest.NewRecorder() + testHandler.CreateHook(w, req) + if w.Code != http.StatusForbidden { + t.Errorf("agent create without principal: status %d, want 403", w.Code) + } +} + +func containsHook(list []HookResponse, id string) bool { + for _, h := range list { + if h.ID == id { + return true + } + } + return false +} diff --git a/server/internal/handler/issue.go b/server/internal/handler/issue.go index 13849c9d69e..7366d1669c9 100644 --- a/server/internal/handler/issue.go +++ b/server/internal/handler/issue.go @@ -18,6 +18,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/logger" "github.com/multica-ai/multica/server/internal/middleware" @@ -2681,18 +2682,105 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { return } } + // Which of the remaining bare-narg columns this request actually targeted. + // An untouched one is rebuilt from the locked row inside the tx (review + // point 1) so a concurrent writer's change is never silently rolled back. + _, touchedStartDate := rawFields["start_date"] + _, touchedDueDate := rawFields["due_date"] + _, touchedParent := rawFields["parent_issue_id"] + _, touchedProject := rawFields["project_id"] + _, touchedStage := rawFields["stage"] attachmentIDs, ok := parseUUIDSliceOrBadRequest(w, req.AttachmentIDs, "attachment_ids") if !ok { return } - issue, err := h.Queries.UpdateIssue(r.Context(), params) + // Resolve the acting identity once, up here, so the transactional-outbox + // events below and the realtime publish further down share one actor. + actorType, actorID := h.resolveActor(r, userID, workspaceID) + actorUUID, _ := util.ParseUUID(actorID) + eventActor := domainevent.ActorFrom(actorType, actorUUID) + + // Transactional outbox (MUL-4332): commit the issue update and any derived + // status_changed / assigned event in one transaction, so a crash can never + // separate the fact from its event. The pre-image is read under a row lock + // INSIDE the tx (not from the pre-tx prevIssue snapshot), so concurrent + // transitions serialize and each records the true edge (review point 3), and + // every untouched nullable column is rebuilt from it (review point 1). + var issue db.Issue + var before db.Issue + err = domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + locked, err := qtx.LockIssueRowForUpdate(r.Context(), db.LockIssueRowForUpdateParams{ + ID: prevIssue.ID, + WorkspaceID: prevIssue.WorkspaceID, + }) + if err != nil { + return nil, err + } + before = locked + + // Rebuild every bare-narg column this request did NOT target from the + // locked row, so an unrelated update never clobbers a field a concurrent + // writer just changed (review point 1). assignee_type/id move together + // because validateAssigneePair validated them as a pair above. + if !touchedType && !touchedID { + params.AssigneeType = before.AssigneeType + params.AssigneeID = before.AssigneeID + } + if !touchedStartDate { + params.StartDate = before.StartDate + } + if !touchedDueDate { + params.DueDate = before.DueDate + } + if !touchedParent { + params.ParentIssueID = before.ParentIssueID + } + if !touchedProject { + params.ProjectID = before.ProjectID + } + if !touchedStage { + params.Stage = before.Stage + } + + updated, err := qtx.UpdateIssue(r.Context(), params) + if err != nil { + return nil, err + } + issue = updated + + var events []domainevent.Event + if before.Status != updated.Status { + events = append(events, domainevent.IssueStatusChanged(updated.WorkspaceID, updated.ID, eventActor, + domainevent.IssueStatusChangedPayload{From: before.Status, To: updated.Status})) + } + // Only emit assigned when the request actually targeted the assignee, so + // an unrelated field update never surfaces a spurious assignment; `from` + // still comes from the locked row. + if (touchedType || touchedID) && + (before.AssigneeType.String != updated.AssigneeType.String || uuidToString(before.AssigneeID) != uuidToString(updated.AssigneeID)) { + events = append(events, domainevent.IssueAssigned(updated.WorkspaceID, updated.ID, eventActor, + domainevent.IssueAssignedPayload{ + FromAssigneeType: before.AssigneeType.String, + FromAssigneeID: uuidToString(before.AssigneeID), + ToAssigneeType: updated.AssigneeType.String, + ToAssigneeID: uuidToString(updated.AssigneeID), + })) + } + return events, nil + }) if err != nil { slog.Warn("update issue failed", append(logger.RequestAttrs(r), "error", err, "issue_id", id, "workspace_id", workspaceID)...) writeError(w, http.StatusInternalServerError, "failed to update issue: "+err.Error()) return } + // The locked pre-image is the authoritative prev-state: drive every + // post-commit side effect (realtime publish, enqueue predicate, parent + // notify) from the transition that TRULY happened rather than the pre-tx + // snapshot, which a concurrent writer may have already superseded (review + // point 4). + prevIssue = before if len(attachmentIDs) > 0 { h.linkAttachmentsByIssueIDs(r.Context(), issue.ID, issue.WorkspaceID, attachmentIDs) @@ -2720,9 +2808,7 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { dueDateChanged := prevDueDate != resp.DueDate && (prevDueDate == nil) != (resp.DueDate == nil) || (prevDueDate != nil && resp.DueDate != nil && *prevDueDate != *resp.DueDate) - // Determine actor identity: agent (via X-Agent-ID header) or member. - actorType, actorID := h.resolveActor(r, userID, workspaceID) - + // actorType / actorID were resolved above (shared with the domain events). h.publish(protocol.EventIssueUpdated, workspaceID, actorType, actorID, map[string]any{ "issue": resp, "assignee_changed": assigneeChanged, @@ -3229,16 +3315,88 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { continue } } + // Untouched bare-narg columns are rebuilt from the locked row inside the + // tx (review point 1) so an unrelated field is never rolled back. + _, batchTouchedStartDate := rawUpdates["start_date"] + _, batchTouchedDueDate := rawUpdates["due_date"] + _, batchTouchedParent := rawUpdates["parent_issue_id"] + _, batchTouchedProject := rawUpdates["project_id"] + _, batchTouchedStage := rawUpdates["stage"] - issue, err := h.Queries.UpdateIssue(r.Context(), params) - if err != nil { - slog.Warn("batch update issue failed", "issue_id", issueID, "error", err) + actorType, actorID := h.resolveActor(r, userID, workspaceID) + batchActorUUID, _ := util.ParseUUID(actorID) + batchEventActor := domainevent.ActorFrom(actorType, batchActorUUID) + + // Transactional outbox (MUL-4332): each row's update and its derived + // status_changed / assigned events commit atomically, per issue. `from` + // is read under a row lock inside the tx so concurrent transitions record + // the true edge (review point 3). + var issue db.Issue + var before db.Issue + if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + locked, err := qtx.LockIssueRowForUpdate(r.Context(), db.LockIssueRowForUpdateParams{ + ID: prevIssue.ID, + WorkspaceID: prevIssue.WorkspaceID, + }) + if err != nil { + return nil, err + } + before = locked + + // Rebuild every untouched bare-narg column from the locked row so a + // concurrent writer's change is never silently rolled back (review + // point 1). See UpdateIssue for the assignee-pair rationale. + if !batchTouchedType && !batchTouchedID { + params.AssigneeType = before.AssigneeType + params.AssigneeID = before.AssigneeID + } + if !batchTouchedStartDate { + params.StartDate = before.StartDate + } + if !batchTouchedDueDate { + params.DueDate = before.DueDate + } + if !batchTouchedParent { + params.ParentIssueID = before.ParentIssueID + } + if !batchTouchedProject { + params.ProjectID = before.ProjectID + } + if !batchTouchedStage { + params.Stage = before.Stage + } + + updatedIssue, err := qtx.UpdateIssue(r.Context(), params) + if err != nil { + return nil, err + } + issue = updatedIssue + var events []domainevent.Event + if before.Status != updatedIssue.Status { + events = append(events, domainevent.IssueStatusChanged(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, + domainevent.IssueStatusChangedPayload{From: before.Status, To: updatedIssue.Status})) + } + if (batchTouchedType || batchTouchedID) && + (before.AssigneeType.String != updatedIssue.AssigneeType.String || uuidToString(before.AssigneeID) != uuidToString(updatedIssue.AssigneeID)) { + events = append(events, domainevent.IssueAssigned(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, + domainevent.IssueAssignedPayload{ + FromAssigneeType: before.AssigneeType.String, + FromAssigneeID: uuidToString(before.AssigneeID), + ToAssigneeType: updatedIssue.AssigneeType.String, + ToAssigneeID: uuidToString(updatedIssue.AssigneeID), + })) + } + return events, nil + }); writeErr != nil { + slog.Warn("batch update issue failed", "issue_id", issueID, "error", writeErr) continue } + // Drive every post-commit side effect from the locked pre-image, the + // transition that truly happened, not the pre-tx snapshot (review point 4). + prevIssue = before prefix := h.getIssuePrefix(r.Context(), issue.WorkspaceID) resp := issueToResponse(issue, prefix) - actorType, actorID := h.resolveActor(r, userID, workspaceID) assigneeChanged := (req.Updates.AssigneeType != nil || req.Updates.AssigneeID != nil) && (prevIssue.AssigneeType.String != issue.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(issue.AssigneeID)) diff --git a/server/internal/handler/issue_child_done.go b/server/internal/handler/issue_child_done.go index b9f0be80c9a..5861efa5ecd 100644 --- a/server/internal/handler/issue_child_done.go +++ b/server/internal/handler/issue_child_done.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" ) @@ -300,16 +301,29 @@ func (h *Handler) postChildDoneComment(ctx context.Context, parent, completed db // author_type='system', author_id=zero UUID. The zero UUID is a valid 16 // byte value and the column is NOT NULL; frontend code should branch on // author_type === 'system' rather than on the UUID value. - comment, err := h.Queries.CreateComment(ctx, db.CreateCommentParams{ - IssueID: parent.ID, - WorkspaceID: parent.WorkspaceID, - AuthorType: "system", - AuthorID: pgtype.UUID{Valid: true}, - Content: content, - Type: "system", - ParentID: pgtype.UUID{Valid: false}, - }) - if err != nil { + // Transactional outbox (MUL-4332 review point 2): the system child-done + // comment and its comment.created event commit in one transaction. + var comment db.Comment + if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, cErr := qtx.CreateComment(ctx, db.CreateCommentParams{ + IssueID: parent.ID, + WorkspaceID: parent.WorkspaceID, + AuthorType: "system", + AuthorID: pgtype.UUID{Valid: true}, + Content: content, + Type: "system", + ParentID: pgtype.UUID{Valid: false}, + }) + if cErr != nil { + return nil, cErr + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, domainevent.SystemActor(), + domainevent.CommentCreatedPayload{ + IssueID: uuidToString(created.IssueID), + AuthorType: "system", + })}, nil + }); err != nil { slog.Warn("child done: create system comment failed", "error", err, "child_id", childID, diff --git a/server/internal/handler/onboarding_shim.go b/server/internal/handler/onboarding_shim.go index f460cc6b57b..0362b190222 100644 --- a/server/internal/handler/onboarding_shim.go +++ b/server/internal/handler/onboarding_shim.go @@ -31,6 +31,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/logger" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" @@ -278,6 +279,12 @@ func (h *Handler) BootstrapOnboardingRuntime(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") return } + // Transactional outbox (MUL-4332): onboarding issue.created. + if _, err := domainevent.Write(r.Context(), qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + slog.Warn("bootstrap onboarding (shim): write issue.created event failed", append(logger.RequestAttrs(r), "error", err)...) + writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") + return + } issueCreated = true } @@ -436,6 +443,12 @@ func (h *Handler) BootstrapOnboardingNoRuntime(w http.ResponseWriter, r *http.Re writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") return } + // Transactional outbox (MUL-4332): onboarding issue.created. + if _, err := domainevent.Write(r.Context(), qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + slog.Warn("bootstrap no-runtime onboarding (shim): write issue.created event failed", append(logger.RequestAttrs(r), "error", err)...) + writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") + return + } issueCreated = true } diff --git a/server/internal/handler/recover_orphans_drain_test.go b/server/internal/handler/recover_orphans_drain_test.go new file mode 100644 index 00000000000..e2605d9ac9a --- /dev/null +++ b/server/internal/handler/recover_orphans_drain_test.go @@ -0,0 +1,209 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +// recover-orphans must DRAIN every orphaned task for a runtime, not just the first +// page (MUL-4332 review round 3, point 1). Registration flips the runtime back +// online, so the offline sweep will not reap anything a single capped call leaves +// behind — the daemon therefore pages until has_more is false. We seed +// orphanRecoveryBatchSize + 1 orphans (one past a single page) and drive the real +// handler exactly as the daemon client does: thread the keyset cursor from each +// response into the next request. All of them must end failed, each with one +// task.failed event, across exactly two pages. +func TestRecoverOrphansDrainsPastBatchLimit(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + + // A dedicated runtime + agent in the handler-test workspace so the daemon token + // (scoped to testWorkspaceID) authorizes recover-orphans for it, isolated from + // the shared fixture runtime. + var runtimeID, agentID string + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent_runtime (workspace_id, name, runtime_mode, provider, status, device_info, metadata, owner_id) + VALUES ($1, 'orphan-drain-runtime', 'cloud', 'codex', 'online', '', '{}'::jsonb, $2) + RETURNING id`, testWorkspaceID, testUserID).Scan(&runtimeID); err != nil { + t.Fatalf("seed runtime: %v", err) + } + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent (workspace_id, name, runtime_mode, runtime_config, runtime_id, visibility, + max_concurrent_tasks, owner_id, instructions, custom_env, custom_args) + VALUES ($1, 'orphan-drain-agent', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3, '', '{}'::jsonb, '[]'::jsonb) + RETURNING id`, testWorkspaceID, runtimeID, testUserID).Scan(&agentID); err != nil { + t.Fatalf("seed agent: %v", err) + } + t.Cleanup(func() { + // agent_runtime → agent → agent_task_queue cascade on delete cleans the rows; + // task.failed events carry no FK, so drop them explicitly first. + testPool.Exec(context.Background(), + `DELETE FROM domain_event WHERE subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, runtimeID) + testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID) + }) + + // Seed one past a single page. attempt=1/max=2 with no issue/chat means the row + // is NOT retry-eligible (retryEligible needs an issue or chat), so the shared + // post-fail pipeline neither auto-retries nor resets an issue — the assertion is + // about pagination alone. A single INSERT gives them all the same created_at, so + // this also exercises the keyset id tiebreaker. + total := orphanRecoveryBatchSize + 1 + if _, err := testPool.Exec(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, status, priority) + SELECT $1, $2, 'running', 0 FROM generate_series(1, $3)`, agentID, runtimeID, total); err != nil { + t.Fatalf("seed orphaned tasks: %v", err) + } + + // Drive the real handler in a drain loop, threading the cursor like the daemon. + // paginate:true is the current daemon's capability signal — without it the server + // treats the caller as a legacy client and drains every page itself in one call + // (that legacy path is covered by TestRecoverOrphansLegacyClientDrainsServerSide). + const daemonID = "recover-drain-test" + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + var body any = map[string]any{"paginate": true} + totalFailed, pages := 0, 0 + for { + pages++ + if pages > 10 { + t.Fatalf("drain did not terminate after %d pages", pages) + } + w := httptest.NewRecorder() + req := withURLParam(newDaemonTokenRequest(http.MethodPost, path, body, testWorkspaceID, daemonID), "runtimeId", runtimeID) + testHandler.RecoverOrphanedTasks(w, req) + if w.Code != http.StatusOK { + t.Fatalf("page %d: status %d: %s", pages, w.Code, w.Body.String()) + } + var resp RecoverOrphansResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("page %d decode: %v", pages, err) + } + totalFailed += resp.Orphaned + if !resp.HasMore { + break + } + if resp.NextCursorCreatedAt == "" || resp.NextCursorID == "" { + t.Fatalf("page %d: has_more=true but empty cursor", pages) + } + body = map[string]any{"paginate": true, "cursor_created_at": resp.NextCursorCreatedAt, "cursor_id": resp.NextCursorID} + } + + if pages != 2 { + t.Errorf("pages = %d, want 2 (a full page of %d, then the final 1)", pages, orphanRecoveryBatchSize) + } + if totalFailed != total { + t.Errorf("failed across drain = %d, want %d", totalFailed, total) + } + + // No orphan is left selectable, and each failed task has exactly one event. + var remaining int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue + WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory')`, + runtimeID).Scan(&remaining); err != nil { + t.Fatalf("count remaining: %v", err) + } + if remaining != 0 { + t.Errorf("remaining orphans = %d, want 0 (drain must reach the row past the first page)", remaining) + } + var events int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM domain_event + WHERE type = 'task.failed' AND subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, + runtimeID).Scan(&events); err != nil { + t.Fatalf("count events: %v", err) + } + if events != total { + t.Errorf("task.failed events = %d, want %d (fact ⇔ event, one per failed row)", events, total) + } +} + +// A LEGACY daemon (one that predates paging) POSTs {} exactly once and ignores the +// response body, so it can never thread the keyset cursor across pages. With 501+ +// orphans on a re-registered runtime — which registration has already flipped back +// `online`, so the offline sweep won't reap the tail — the server must therefore +// drain every page itself in one call (MUL-4332 review round 4, point 2). We seed +// orphanRecoveryBatchSize + 1 orphans, POST a single {} request with NO paginate +// capability, and assert all of them end failed with one event each — none leak past +// the first server page. +func TestRecoverOrphansLegacyClientDrainsServerSide(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + + var runtimeID, agentID string + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent_runtime (workspace_id, name, runtime_mode, provider, status, device_info, metadata, owner_id) + VALUES ($1, 'orphan-legacy-runtime', 'cloud', 'codex', 'online', '', '{}'::jsonb, $2) + RETURNING id`, testWorkspaceID, testUserID).Scan(&runtimeID); err != nil { + t.Fatalf("seed runtime: %v", err) + } + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent (workspace_id, name, runtime_mode, runtime_config, runtime_id, visibility, + max_concurrent_tasks, owner_id, instructions, custom_env, custom_args) + VALUES ($1, 'orphan-legacy-agent', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3, '', '{}'::jsonb, '[]'::jsonb) + RETURNING id`, testWorkspaceID, runtimeID, testUserID).Scan(&agentID); err != nil { + t.Fatalf("seed agent: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), + `DELETE FROM domain_event WHERE subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, runtimeID) + testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID) + }) + + // One past a single server page (see TestRecoverOrphansDrainsPastBatchLimit for + // why attempt/max with no issue keeps the post-fail pipeline a no-op here). + total := orphanRecoveryBatchSize + 1 + if _, err := testPool.Exec(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, status, priority) + SELECT $1, $2, 'running', 0 FROM generate_series(1, $3)`, agentID, runtimeID, total); err != nil { + t.Fatalf("seed orphaned tasks: %v", err) + } + + // Legacy request: {} body, no paginate capability, called exactly once. + const daemonID = "recover-legacy-test" + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + w := httptest.NewRecorder() + req := withURLParam(newDaemonTokenRequest(http.MethodPost, path, map[string]any{}, testWorkspaceID, daemonID), "runtimeId", runtimeID) + testHandler.RecoverOrphanedTasks(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var resp RecoverOrphansResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.HasMore { + t.Errorf("legacy drain returned has_more=true; the server must fully drain in one call (a legacy client cannot page)") + } + if resp.Orphaned != total { + t.Errorf("orphaned = %d, want %d (single legacy call must drain past the first server page)", resp.Orphaned, total) + } + + var remaining int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue + WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory')`, + runtimeID).Scan(&remaining); err != nil { + t.Fatalf("count remaining: %v", err) + } + if remaining != 0 { + t.Errorf("remaining orphans = %d, want 0 (legacy client can't page, so nothing may be left behind)", remaining) + } + var events int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM domain_event + WHERE type = 'task.failed' AND subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, + runtimeID).Scan(&events); err != nil { + t.Fatalf("count events: %v", err) + } + if events != total { + t.Errorf("task.failed events = %d, want %d (fact ⇔ event, one per failed row)", events, total) + } +} diff --git a/server/internal/handler/task_lifecycle.go b/server/internal/handler/task_lifecycle.go index e84abcb4200..4e1ed6abd66 100644 --- a/server/internal/handler/task_lifecycle.go +++ b/server/internal/handler/task_lifecycle.go @@ -1,11 +1,13 @@ package handler import ( + "context" "encoding/json" "errors" "io" "log/slog" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" @@ -13,48 +15,234 @@ import ( db "github.com/multica-ai/multica/server/pkg/db/generated" ) -// RecoverOrphanedTasks is called by the daemon at startup for each runtime -// it owns. It atomically fails any dispatched/running tasks the server still -// believes belong to that runtime — those are the tasks the previous daemon -// process was running when it died — and triggers MaybeRetryFailedTask for -// each so the user sees a fresh attempt instead of a permanently stuck row. +// orphanRecoveryBatchSize bounds how many orphaned tasks one recover-orphans +// PAGE fails (MUL-4332 review point 2). Registration upserts the runtime back to +// `online`, so the every-tick offline sweep will NOT reap anything this call leaves +// behind (review round 3, point 1); the daemon therefore drains all pages via the +// keyset cursor rather than relying on a follow-up sweep. This bound only caps the +// lock hold and event volume per page. +const orphanRecoveryBatchSize = 500 + +// maxServerOrphanDrainPages bounds the server-side drain loop used for legacy +// clients (see RecoverOrphanedTasks). Each page fails up to orphanRecoveryBatchSize +// (500) rows, so this covers ~500k orphaned rows in one request — far beyond any +// real restart. It is purely a backstop so a pathological backlog can never hold +// one HTTP request open forever; any residual is reclaimed on the next +// registration. It mirrors the daemon client's own maxRecoverOrphanPages backstop. +const maxServerOrphanDrainPages = 1000 + +// RecoverOrphansRequest is the optional body of POST recover-orphans. +// +// Paginate is the client's capability signal (MUL-4332 review round 4, point 2): +// a current daemon sets it true and drives the drain itself, threading the keyset +// cursor across calls. A legacy daemon predates paging — it POSTs `{}` (or an empty +// body) exactly once and ignores the response — so Paginate is absent/false and the +// server must instead drain every page itself, or orphans past the first page leak +// on a re-registered (now `online`) runtime that the offline sweep won't reap. // -// This is the targeted fix for "issue stuck at in_progress when daemon -// restarts mid-task": the runtime heartbeat sweeper takes up to 75s + the -// in-process task timeout (2.5h) to notice such tasks; the daemon itself -// knows the moment it comes back up, so we let it report orphan recovery. +// Cursor* is threaded back by a paginating client for each subsequent page. Both +// fields move together — a partial cursor is rejected. +type RecoverOrphansRequest struct { + Paginate bool `json:"paginate,omitempty"` + CursorCreatedAt string `json:"cursor_created_at,omitempty"` + CursorID string `json:"cursor_id,omitempty"` +} + +// RecoverOrphansResponse reports one page of recovery. HasMore + NextCursor* let the +// daemon drain the runtime's orphans across pages; NextCursor* advances over every +// candidate this page locked (failed OR skipped-poison), so a page of poison at the +// front cannot pin the drain — the next page steps past it (review round 3, point 1). +type RecoverOrphansResponse struct { + Orphaned int `json:"orphaned"` + Retried int `json:"retried"` + Skipped int `json:"skipped"` + HasMore bool `json:"has_more"` + NextCursorCreatedAt string `json:"next_cursor_created_at,omitempty"` + NextCursorID string `json:"next_cursor_id,omitempty"` +} + +// RecoverOrphanedTasks is called by the daemon at startup for each runtime it owns. +// It atomically fails a bounded page of the dispatched/running tasks the server +// still believes belong to that runtime — those the previous daemon process was +// running when it died — emitting each row's task.failed event in the same +// transaction, then runs the shared post-failure pipeline (auto-retry, issue +// rollback, reconcile). The daemon calls it repeatedly, threading the keyset cursor, +// until HasMore is false. +// +// This is the targeted fix for "issue stuck at in_progress when daemon restarts +// mid-task": the runtime heartbeat sweeper takes up to 75s + the in-process task +// timeout (2.5h) to notice such tasks; the daemon knows the moment it comes back up. +// Because registration flips the runtime back online, the offline sweep will not +// catch a row beyond this page (review round 3, point 1) — hence the cursor-driven +// drain instead of a single capped call. func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") if _, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID); !ok { return } - rows, err := h.Queries.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID)) - if err != nil { - slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) - writeError(w, http.StatusInternalServerError, "recover orphans failed") + // Optional keyset cursor: absent on the first page, threaded back by the daemon + // for each subsequent page. + var req RecoverOrphansRequest + if r.ContentLength != 0 { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil && err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + afterCreatedAt, afterID, ok := parseOrphanCursor(w, req) + if !ok { + return + } + + runtimeUUID := parseUUID(runtimeID) + + // A capability-aware client (req.Paginate) drives the drain itself: fail exactly + // one page and hand back the cursor for it to continue. A legacy client — which + // POSTs {} once and ignores the body — cannot page, so the server drains every + // page itself in bounded per-page transactions; otherwise 501+ orphans on a + // re-registered (now `online`) runtime leak permanently (MUL-4332 review round 4, + // point 2). Both modes share recoverOrphansPage, so the per-page fact ⇔ event and + // poison-isolation semantics are identical. + if req.Paginate { + resp, _, _, err := h.recoverOrphansPage(r.Context(), runtimeUUID, runtimeID, afterCreatedAt, afterID) + if err != nil { + slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) + writeError(w, http.StatusInternalServerError, "recover orphans failed") + return + } + writeJSON(w, http.StatusOK, resp) return } + // Legacy single-shot client: drain all pages server-side. A partial cursor is + // already rejected above, so a legacy client always starts from the first page + // (both cursor components NULL). + var agg RecoverOrphansResponse + cursorCreatedAt, cursorID := afterCreatedAt, afterID + for page := 0; page < maxServerOrphanDrainPages; page++ { + resp, nextCreatedAt, nextID, err := h.recoverOrphansPage(r.Context(), runtimeUUID, runtimeID, cursorCreatedAt, cursorID) + if err != nil { + slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) + writeError(w, http.StatusInternalServerError, "recover orphans failed") + return + } + agg.Orphaned += resp.Orphaned + agg.Retried += resp.Retried + agg.Skipped += resp.Skipped + if !resp.HasMore { + break + } + cursorCreatedAt, cursorID = nextCreatedAt, nextID + } + // The legacy client ignores the body, but return an accurate, explicitly + // non-paged (has_more=false) aggregate for logs and proxies. + writeJSON(w, http.StatusOK, agg) +} + +// recoverOrphansPage fails one bounded page of the runtime's orphaned tasks and +// runs the shared post-failure pipeline. It returns the page result plus the raw +// keyset cursor of the last candidate (for the server-side legacy drain to thread +// without re-parsing its own RFC3339 string). The cursor advances over EVERY +// candidate the page locked — failed OR skipped-poison — so neither a paginating +// client nor the server-side drain can be pinned by a page of poison at the front. +func (h *Handler) recoverOrphansPage( + ctx context.Context, + runtimeUUID pgtype.UUID, + runtimeID string, + afterCreatedAt pgtype.Timestamptz, + afterID pgtype.UUID, +) (RecoverOrphansResponse, pgtype.Timestamptz, pgtype.UUID, error) { + // Select one page of the runtime's orphaned tasks and fail the resolvable ones + // with their task.failed events atomically (MUL-4332 review point 2), so + // recovery converges onto the outbox like the runtime sweepers and an + // unresolvable poison row cannot block the rest. We capture the raw candidate + // page (before poison isolation drops rows) to compute has_more and the cursor. + var candidates []db.AgentTaskQueue + rows, err := h.TaskService.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: runtimeUUID, + AfterCreatedAt: afterCreatedAt, + AfterID: afterID, + MaxPerTick: orphanRecoveryBatchSize, + }) + candidates = c + return c, e + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) + if err != nil { + return RecoverOrphansResponse{}, pgtype.Timestamptz{}, pgtype.UUID{}, err + } + // Funnel through the shared post-failure pipeline so we get the same // task:failed events, agent reconcile, issue rollback, and auto-retry // behaviour as the runtime sweeper. This was previously a fast-path // that bypassed those side effects, leaving the UI stale when no retry // was created (max_attempts exhausted, autopilot, non-retryable reason). - retried := h.TaskService.HandleFailedTasks(r.Context(), rows) + retried := h.TaskService.HandleFailedTasks(ctx, rows) - if len(rows) > 0 { - slog.Info("recover-orphans completed", + // A full candidate page means there may be more behind it. The cursor advances + // over the LAST candidate we locked (failed or skipped), so the next page never + // re-selects this page — including any poison rows left un-failed at the front. + resp := RecoverOrphansResponse{ + Orphaned: len(rows), + Retried: retried, + Skipped: len(candidates) - len(rows), + HasMore: len(candidates) == orphanRecoveryBatchSize, + } + var nextCreatedAt pgtype.Timestamptz + var nextID pgtype.UUID + if resp.HasMore && len(candidates) > 0 { + last := candidates[len(candidates)-1] + nextCreatedAt = last.CreatedAt + nextID = last.ID + resp.NextCursorCreatedAt = last.CreatedAt.Time.UTC().Format(time.RFC3339Nano) + resp.NextCursorID = uuidToString(last.ID) + } + + if len(candidates) > 0 { + slog.Info("recover-orphans page", "runtime_id", runtimeID, - "orphaned", len(rows), + "candidates", len(candidates), + "orphaned", resp.Orphaned, + "skipped", resp.Skipped, "retried", retried, + "has_more", resp.HasMore, ) } - writeJSON(w, http.StatusOK, map[string]any{ - "orphaned": len(rows), - "retried": retried, - }) + return resp, nextCreatedAt, nextID, nil +} + +// parseOrphanCursor turns the optional (cursor_created_at, cursor_id) request pair +// into keyset params. Neither present → the first page (both NULL). Both present → +// the keyset. Exactly one present is a malformed cursor and is rejected, so a bad +// client can never silently restart the drain from the beginning (an infinite loop). +func parseOrphanCursor(w http.ResponseWriter, req RecoverOrphansRequest) (pgtype.Timestamptz, pgtype.UUID, bool) { + if req.CursorCreatedAt == "" && req.CursorID == "" { + return pgtype.Timestamptz{}, pgtype.UUID{}, true + } + if req.CursorCreatedAt == "" || req.CursorID == "" { + writeError(w, http.StatusBadRequest, "cursor_created_at and cursor_id must be set together") + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + ts, err := time.Parse(time.RFC3339Nano, req.CursorCreatedAt) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid cursor_created_at") + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + id, ok := parseUUIDOrBadRequest(w, req.CursorID, "cursor_id") + if !ok { + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + return pgtype.Timestamptz{Time: ts, Valid: true}, id, true } // PinTaskSession lets the daemon persist the agent's session_id and diff --git a/server/internal/service/autopilot.go b/server/internal/service/autopilot.go index 03831e0e168..c3d0373a7c1 100644 --- a/server/internal/service/autopilot.go +++ b/server/internal/service/autopilot.go @@ -16,6 +16,7 @@ import ( "github.com/multica-ai/multica/server/internal/analytics" "github.com/multica-ai/multica/server/internal/attribution" "github.com/multica-ai/multica/server/internal/dispatch" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/issueposition" @@ -676,6 +677,12 @@ func (s *AutopilotService) dispatchCreateIssue(ctx context.Context, ap db.Autopi } *run = updatedRun + // Transactional outbox (MUL-4332): emit issue.created atomically with the + // autopilot-dispatched issue insert, same as the HTTP create path. + if _, err := domainevent.Write(ctx, qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + return fmt.Errorf("write issue.created event: %w", err) + } + if err := tx.Commit(ctx); err != nil { return fmt.Errorf("commit tx: %w", err) } diff --git a/server/internal/service/domain_event_bulk_fail_test.go b/server/internal/service/domain_event_bulk_fail_test.go new file mode 100644 index 00000000000..5e06e697873 --- /dev/null +++ b/server/internal/service/domain_event_bulk_fail_test.go @@ -0,0 +1,267 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// A single unresolvable "poison" row must not fail-close the whole bulk sweep +// (MUL-4332 review point 2): the resolvable tasks still commit their fact + event +// atomically, and the poison row is skipped (left for ops), not failed. +func TestFailBulkTasksIsolatesPoisonRow(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + seedTask := func() string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + goodID := seedTask() + poisonID := seedTask() + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, []string{goodID, poisonID}) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, []string{goodID, poisonID}) + }) + + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + good, e := qtx.GetAgentTask(ctx, util.MustParseUUID(goodID)) + if e != nil { + return nil, e + } + poison, e := qtx.GetAgentTask(ctx, util.MustParseUUID(poisonID)) + if e != nil { + return nil, e + } + // A real agent_task_queue row always resolves (agent_id is NOT NULL + + // FK), so we present the second candidate as a VIEW with its resolvable + // links stripped — exercising the defensive isolation path directly. The + // DB row (poisonID) is untouched and must be left un-failed. + poison.AgentID = pgtype.UUID{} + poison.IssueID = pgtype.UUID{} + poison.ChatSessionID = pgtype.UUID{} + poison.AutopilotRunID = pgtype.UUID{} + return []db.AgentTaskQueue{good, poison}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("FailBulkTasksWithEvents returned an error — a poison row must not fail the batch: %v", err) + } + + // Only the resolvable task is failed and returned. + if len(failed) != 1 || util.UUIDToString(failed[0].ID) != goodID { + t.Fatalf("expected only the resolvable task returned as failed, got %+v", failed) + } + if s := taskStatusForTest(t, pool, goodID); s != "failed" { + t.Errorf("resolvable task status = %q, want failed", s) + } + if n := subjectEventCount(t, pool, goodID); n != 1 { + t.Errorf("resolvable task events = %d, want 1", n) + } + // The poison row is untouched: not failed, no event. + if s := taskStatusForTest(t, pool, poisonID); s != "running" { + t.Errorf("poison task status = %q, want running (must be left for ops, not failed)", s) + } + if n := subjectEventCount(t, pool, poisonID); n != 0 { + t.Errorf("poison task events = %d, want 0 (no valid event is possible without a workspace)", n) + } +} + +// A transient failure inside the fail transaction must roll the whole batch back +// — no fact is committed without its event — so the task stays selectable and the +// next sweep tick reclaims it (MUL-4332 review point 2). +func TestFailBulkTasksTransientFailureRecoversNextTick(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + sel := func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, e := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if e != nil { + return nil, e + } + return []db.AgentTaskQueue{row}, nil + } + + // Tick 1: the fail step errors (a transient DB blip). The whole batch rolls + // back — the task is NOT failed and no event is written. + boom := errors.New("transient DB blip") + if _, err := svc.FailBulkTasksWithEvents(ctx, sel, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return nil, boom + }); !errors.Is(err, boom) { + t.Fatalf("expected the transient error to surface, got %v", err) + } + if s := taskStatusForTest(t, pool, taskID); s != "running" { + t.Fatalf("after rollback task status = %q, want running (no fact may commit)", s) + } + if n := subjectEventCount(t, pool, taskID); n != 0 { + t.Fatalf("after rollback events = %d, want 0", n) + } + + // Tick 2: the same still-selectable task fails cleanly. + failed, err := svc.FailBulkTasksWithEvents(ctx, sel, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("retry FailBulkTasksWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("retry expected 1 failed task, got %d", len(failed)) + } + if s := taskStatusForTest(t, pool, taskID); s != "failed" { + t.Errorf("after retry task status = %q, want failed", s) + } + if n := subjectEventCount(t, pool, taskID); n != 1 { + t.Errorf("after retry events = %d, want 1", n) + } +} + +// The stuck-issue reset must re-decide UNDER the row lock: if a user moves the +// issue to a terminal status in the window between the pre-tx status read and the +// reset lock, the sweeper must not reopen it (MUL-4332 review point 4). We force +// exactly that interleaving by holding the row lock with an uncommitted move to +// 'done' until the reset is blocked on the lock. +func TestHandleFailedTasksDoesNotReopenUserCompletedIssue(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + if _, err := pool.Exec(ctx, `UPDATE issue SET status = 'in_progress' WHERE id = $1`, issueID); err != nil { + t.Fatalf("set issue in_progress: %v", err) + } + // A non-retryable failure so no auto-retry short-circuits the reset branch. + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, failure_reason, priority, completed_at) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'failed', 'agent_error', 0, now()) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed failed task: %v", err) + } + failedTask, err := queries.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if err != nil { + t.Fatalf("load failed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // Holder: lock the issue row and move it to 'done' (uncommitted). A plain + // read (HandleFailedTasks' pre-tx GetIssue) still sees the committed + // 'in_progress', so it enters the reset branch — then blocks on this lock. + locked := make(chan struct{}) + release := make(chan struct{}) + go func() { + tx, e := pool.Begin(context.Background()) + if e != nil { + t.Errorf("holder begin: %v", e) + close(locked) + return + } + defer tx.Rollback(context.Background()) + if _, e := tx.Exec(context.Background(), `SELECT status FROM issue WHERE id = $1 FOR UPDATE`, issueID); e != nil { + t.Errorf("holder lock: %v", e) + close(locked) + return + } + if _, e := tx.Exec(context.Background(), `UPDATE issue SET status = 'done' WHERE id = $1`, issueID); e != nil { + t.Errorf("holder update: %v", e) + close(locked) + return + } + close(locked) + <-release + tx.Commit(context.Background()) + }() + + <-locked + done := make(chan struct{}) + go func() { + svc.HandleFailedTasks(ctx, []db.AgentTaskQueue{failedTask}) + close(done) + }() + // Let HandleFailedTasks read the in_progress snapshot and block on the reset + // lock, then let the user's 'done' commit win the row. + time.Sleep(400 * time.Millisecond) + close(release) + <-done + + if s := issueStatusForTest(t, pool, issueID); s != "done" { + t.Fatalf("issue status = %q, want done — a user-completed issue must not be reopened by the stuck-issue reset (review point 4)", s) + } +} + +func taskStatusForTest(t *testing.T, pool *pgxpool.Pool, taskID string) string { + t.Helper() + var s string + if err := pool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&s); err != nil { + t.Fatalf("query task status: %v", err) + } + return s +} + +func issueStatusForTest(t *testing.T, pool *pgxpool.Pool, issueID string) string { + t.Helper() + var s string + if err := pool.QueryRow(context.Background(), `SELECT status FROM issue WHERE id = $1`, issueID).Scan(&s); err != nil { + t.Fatalf("query issue status: %v", err) + } + return s +} + +func subjectEventCount(t *testing.T, pool *pgxpool.Pool, subjectID string) int { + t.Helper() + var n int + if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM domain_event WHERE subject_id = $1`, subjectID).Scan(&n); err != nil { + t.Fatalf("count events: %v", err) + } + return n +} diff --git a/server/internal/service/domain_event_task_failed_test.go b/server/internal/service/domain_event_task_failed_test.go new file mode 100644 index 00000000000..987b3069aa1 --- /dev/null +++ b/server/internal/service/domain_event_task_failed_test.go @@ -0,0 +1,188 @@ +package service + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// FailBulkTasksWithEvents is the shared mechanism behind every bulk task.failed +// path (the runtime sweepers + daemon orphan recovery). Failing a task through it +// must persist a task.failed domain event atomically with the fail (MUL-4332 +// review point 2), stamped with the resolved workspace and issue, attributed to +// the platform (SystemActor, not the agent) and carrying a retryable flag that +// agrees with the auto-retry decision (review point 3). +func TestFailBulkTasksWithEventsEmitsTaskFailed(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + // Seed a running task for that agent/issue. + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed running task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, gerr := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if gerr != nil { + return nil, gerr + } + return []db.AgentTaskQueue{row}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("FailBulkTasksWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("expected 1 failed task, got %d", len(failed)) + } + + // Exactly one task.failed event for the task, carrying the issue + error, + // attributed to the platform, with a retry_eligible flag matching the shared + // retryEligible predicate. + var evtType, actorType, payload string + var retryEligibleFlag bool + if err := pool.QueryRow(ctx, + `SELECT type, actor_type, (payload->>'retry_eligible')::bool, payload::text + FROM domain_event WHERE subject_type = 'task' AND subject_id = $1`, + taskID).Scan(&evtType, &actorType, &retryEligibleFlag, &payload); err != nil { + t.Fatalf("expected a task.failed domain event: %v", err) + } + if evtType != "task.failed" { + t.Errorf("type = %q, want task.failed", evtType) + } + if actorType != "system" { + t.Errorf("actor_type = %q, want system (a sweeper fail is platform-driven, not the agent's action)", actorType) + } + if want := retryEligible("runtime_offline", failed[0]); retryEligibleFlag != want { + t.Errorf("retry_eligible = %v, want %v (event must agree with the eligibility predicate)", retryEligibleFlag, want) + } + if !strings.Contains(payload, issueID) { + t.Errorf("payload %s should carry issue_id %s", payload, issueID) + } + if !strings.Contains(payload, "runtime_offline") { + t.Errorf("payload %s should carry the failure reason", payload) + } +} + +// task.failed.retry_eligible is an atomically-decidable eligibility fact committed +// WITH the fail — never a promise that a retry child exists (Elon review round 3, +// point 2). On the bulk sweeper / orphan-recovery paths the event commits inside +// FailBulkTasksWithEvents while the retry child is only created best-effort AFTER +// commit by HandleFailedTasks, where CreateRetryTask can error or the process can +// crash. We reproduce that crash window by committing the fail+event and then NOT +// running the post-commit retry step: the flag must still equal the eligibility +// predicate, the task must be terminal, and no retry child may exist — so a consumer +// can never read retry_eligible as "a fresh attempt is guaranteed to arrive". +func TestTaskFailedRetryEligibleIsNotAChildPromise(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + cases := []struct { + name string + attempt int + maxAttempts int + wantEligible bool + }{ + // A within-budget, infra-shaped failure IS eligible — yet the crash window + // means no child exists after commit until HandleFailedTasks runs. + {"within budget → eligible, still no child in the window", 1, 2, true}, + // A budget-exhausted failure is NOT eligible: the flag is decidable both + // ways, it is not hard-coded true. + {"budget exhausted → not eligible", 2, 2, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, attempt, max_attempts) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0, $3, $4) + RETURNING id`, agentID, issueID, tc.attempt, tc.maxAttempts).Scan(&taskID); err != nil { + t.Fatalf("seed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1 OR parent_task_id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, gerr := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if gerr != nil { + return nil, gerr + } + return []db.AgentTaskQueue{row}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("FailBulkTasksWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("expected 1 failed task, got %d", len(failed)) + } + + // Deliberately DO NOT call HandleFailedTasks: that post-commit step is + // where the retry child would be created. Skipping it models a crash in + // exactly that window. + + // The committed event's flag equals the eligibility predicate... + var got bool + if err := pool.QueryRow(ctx, + `SELECT (payload->>'retry_eligible')::bool FROM domain_event + WHERE subject_type = 'task' AND subject_id = $1`, taskID).Scan(&got); err != nil { + t.Fatalf("expected a task.failed domain event: %v", err) + } + if got != tc.wantEligible { + t.Errorf("retry_eligible = %v, want %v", got, tc.wantEligible) + } + // ...the subject task is terminal... + if s := taskStatusForTest(t, pool, taskID); s != "failed" { + t.Errorf("task status = %q, want failed (task.failed is terminal for the subject)", s) + } + // ...and no retry child exists — the flag is not a child promise. + var children int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue WHERE parent_task_id = $1`, taskID).Scan(&children); err != nil { + t.Fatalf("count retry children: %v", err) + } + if children != 0 { + t.Errorf("retry children = %d, want 0 (no child is created in the crash window)", children) + } + }) + } +} diff --git a/server/internal/service/domain_event_task_workspace_test.go b/server/internal/service/domain_event_task_workspace_test.go new file mode 100644 index 00000000000..425c92d5619 --- /dev/null +++ b/server/internal/service/domain_event_task_workspace_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// resolveTaskWorkspaceForEvent must be fail-closed (MUL-4332 review point 4): a +// task with no resolvable attribution returns an invalid UUID, which the +// task-terminal callers treat as an error and roll the transition back rather +// than committing a fact with no event. It must also succeed via the agent +// fallback for a task that carries only an agent. +func TestResolveTaskWorkspaceForEventFailClosed(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + // Unresolvable: no issue / chat / autopilot / quick-create link and a bogus + // agent id that matches no row → invalid workspace. + orphan := db.AgentTaskQueue{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + AgentID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + } + if ws := svc.resolveTaskWorkspaceForEvent(ctx, queries, orphan); ws.Valid { + t.Fatalf("expected unresolvable workspace to be invalid, got %s", util.UUIDToString(ws)) + } + + // Resolvable via the agent fallback: a task carrying a real agent resolves to + // that agent's workspace. + agentID := createClaimCapacityFixture(t, ctx, pool) + task := db.AgentTaskQueue{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + AgentID: util.MustParseUUID(agentID), + } + if ws := svc.resolveTaskWorkspaceForEvent(ctx, queries, task); !ws.Valid { + t.Fatalf("expected agent fallback to resolve a workspace for agent %s", agentID) + } +} diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go new file mode 100644 index 00000000000..3b5c091d298 --- /dev/null +++ b/server/internal/service/hook.go @@ -0,0 +1,712 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/admission" + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Hook CRUD errors surfaced to the handler for status mapping. Validation +// problems (shape or unresolvable/forbidden target) flow through as +// *automation.ValidationError (→ 400). +var ( + ErrHookNotFound = errors.New("hook not found") + ErrHookSystemManaged = errors.New("system-managed hooks cannot be modified through this API") + ErrHookNoPrincipal = errors.New("no accountable authorization principal for this hook") + ErrHookForbidden = errors.New("only the hook's principal or a workspace admin may modify it") + ErrHookPrincipalDeparted = errors.New("the hook's authorization principal is no longer a member of this workspace") +) + +// HookAuthor carries the resolved identity for a hook write: who is acting +// (creator, pure audit) and the accountable human whose authority the hook runs +// under (§8). Membership and role are NOT carried here — the service re-derives +// them inside the write transaction so a stale snapshot can never authorize a +// write (review round 3, point 3). +type HookAuthor struct { + ActorType string // member | agent + ActorID pgtype.UUID + PrincipalUserID pgtype.UUID +} + +// HookWithRevision pairs a hook row with its active revision so the handler can +// render one complete view. The service returns db rows; the handler shapes JSON. +type HookWithRevision struct { + Hook db.Hook + Revision db.HookRevision +} + +// HookService is the store-only policy layer for Event Hooks (MUL-4332 PR2). +// It validates and persists hook specifications and their immutable revisions; +// it performs no matching or execution. Behaviour is gated at the handler by the +// automation_event_hooks feature flag, so creating hooks changes nothing at +// runtime until the executor slice ships and the flag is enabled. +type HookService struct { + Queries *db.Queries + TxStarter TxStarter +} + +func NewHookService(q *db.Queries, tx TxStarter) *HookService { + return &HookService{Queries: q, TxStarter: tx} +} + +// CreateHook validates the spec (shape + workspace-scoped, principal-gated +// targets) and inserts the hook together with revision #1 in one transaction. +// The writer's membership and every target admission are (re)checked inside that +// transaction against the accountable principal, so an illegal configuration +// never enters the store (§13) and a stale role/membership snapshot cannot +// authorize the write. The two rows reference each other, so both ids are +// generated up front. +func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { + if err := automation.Validate(spec); err != nil { + return HookWithRevision{}, err + } + if !author.PrincipalUserID.Valid { + return HookWithRevision{}, ErrHookNoPrincipal + } + scopeType, scopeID, err := resolveScope(spec.Scope) + if err != nil { + return HookWithRevision{}, err + } + match, conditions, actions, err := marshalRevisionConfig(spec) + if err != nil { + return HookWithRevision{}, err + } + + hookID := util.NewUUID() + revisionID := util.NewUUID() + + var out HookWithRevision + err = s.inTx(ctx, func(qtx *db.Queries) error { + // The creator's principal must be a current member of the workspace. The + // FOR SHARE lock blocks a concurrent removal/demotion from committing before + // this hook write does (review round 5). + creator, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ + UserID: author.PrincipalUserID, WorkspaceID: workspaceID, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrHookNoPrincipal + } + return err + } + // A newly created hook runs under the creator's authority. + if err := validateTargets(ctx, qtx, workspaceID, spec, creator); err != nil { + return err + } + hook, err := qtx.CreateHook(ctx, db.CreateHookParams{ + ID: hookID, + WorkspaceID: workspaceID, + Name: spec.Name, + Enabled: true, + ActiveRevisionID: revisionID, + ScopeType: scopeType, + ScopeID: scopeID, + Origin: "user", + CreatorActorType: author.ActorType, + CreatorActorID: author.ActorID, + AuthorizationPrincipalUserID: author.PrincipalUserID, + }) + if err != nil { + return err + } + rev, err := qtx.CreateHookRevision(ctx, db.CreateHookRevisionParams{ + ID: revisionID, + HookID: hookID, + Revision: 1, + EventType: spec.When.Event, + Match: match, + Conditions: conditions, + FireMode: spec.Fire.Mode, + Actions: actions, + CreatedByType: author.ActorType, + CreatedByID: author.ActorID, + }) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + if err != nil { + return HookWithRevision{}, err + } + return out, nil +} + +// UpdateHook appends a new immutable revision from the spec and repoints the +// hook's active revision (§5.1), all inside one transaction that first locks the +// hook row (so concurrent PATCHes serialize and MAX(revision)+1 cannot collide), +// then re-checks archived/origin, the editor's live membership/role, and the +// edit-authorization gate. Target admission is judged against the hook's LOCKED +// stored principal — never the editor — so an admin editing another member's +// hook can only change configuration, never grant the stored principal reach it +// lacks (review round 3, point 1). Scope is immutable. +func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { + if err := automation.Validate(spec); err != nil { + return HookWithRevision{}, err + } + match, conditions, actions, err := marshalRevisionConfig(spec) + if err != nil { + return HookWithRevision{}, err + } + + revisionID := util.NewUUID() + var out HookWithRevision + err = s.inTx(ctx, func(qtx *db.Queries) error { + existing, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author) + if err != nil { + return err + } + // Editing re-arms the hook, so its STORED principal must still be a live + // member — an admin cannot re-point a departed principal's hook at new + // targets (review round 4, point 1). Admission then runs against that + // principal, never the editor. + principal, err := s.requireLivePrincipal(ctx, qtx, workspaceID, existing.AuthorizationPrincipalUserID) + if err != nil { + return err + } + if err := validateTargets(ctx, qtx, workspaceID, spec, principal); err != nil { + return err + } + maxRev, err := qtx.GetMaxHookRevision(ctx, hookID) + if err != nil { + return err + } + rev, err := qtx.CreateHookRevision(ctx, db.CreateHookRevisionParams{ + ID: revisionID, + HookID: hookID, + Revision: maxRev + 1, + EventType: spec.When.Event, + Match: match, + Conditions: conditions, + FireMode: spec.Fire.Mode, + Actions: actions, + CreatedByType: author.ActorType, + CreatedByID: author.ActorID, + }) + if err != nil { + return err + } + hook, err := qtx.SetHookActiveRevision(ctx, db.SetHookActiveRevisionParams{ + ID: hookID, + WorkspaceID: workspaceID, + ActiveRevisionID: revisionID, + Name: spec.Name, + }) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + if err != nil { + return HookWithRevision{}, err + } + return out, nil +} + +// GetHook loads a hook and its active revision. +func (s *HookService) GetHook(ctx context.Context, workspaceID, hookID pgtype.UUID) (HookWithRevision, error) { + hook, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return HookWithRevision{}, ErrHookNotFound + } + return HookWithRevision{}, err + } + if hook.ArchivedAt.Valid { + return HookWithRevision{}, ErrHookNotFound + } + rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return HookWithRevision{}, err + } + return HookWithRevision{Hook: hook, Revision: rev}, nil +} + +// ListHooks returns every non-archived hook in the workspace with its active +// revision. Hook counts per workspace are small (guardrails cap fan-out, not the +// number of rules), so the per-hook revision lookup is acceptable. +func (s *HookService) ListHooks(ctx context.Context, workspaceID pgtype.UUID) ([]HookWithRevision, error) { + hooks, err := s.Queries.ListHooksByWorkspace(ctx, workspaceID) + if err != nil { + return nil, err + } + out := make([]HookWithRevision, 0, len(hooks)) + for _, hook := range hooks { + rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return nil, err + } + out = append(out, HookWithRevision{Hook: hook, Revision: rev}) + } + return out, nil +} + +// SetEnabled enables/disables a hook. Disable only blocks future matches; it +// does not cancel queued/running executions (§5.1). Load, authorization and the +// mutation all happen inside one transaction against the locked row. +func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype.UUID, enabled bool, reason string, author HookAuthor) (HookWithRevision, error) { + disabledReason := pgtype.Text{} + if !enabled && reason != "" { + disabledReason = pgtype.Text{String: reason, Valid: true} + } + var out HookWithRevision + err := s.inTx(ctx, func(qtx *db.Queries) error { + existing, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author) + if err != nil { + return err + } + // Enable re-arms the hook, so (like update) its stored principal must still + // be a live member. Disable is a degrading op: an admin may safely disable a + // departed principal's hook (review round 4, point 1). + if enabled { + if _, err := s.requireLivePrincipal(ctx, qtx, workspaceID, existing.AuthorizationPrincipalUserID); err != nil { + return err + } + } + hook, err := qtx.SetHookEnabled(ctx, db.SetHookEnabledParams{ + ID: hookID, + WorkspaceID: workspaceID, + Enabled: enabled, + DisabledReason: disabledReason, + }) + if err != nil { + return err + } + rev, err := qtx.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + return out, err +} + +// ArchiveHook soft-deletes a hook (§5.1); revisions/executions/effects are kept. +// Load, authorization and the mutation all happen inside one transaction. +func (s *HookService) ArchiveHook(ctx context.Context, workspaceID, hookID pgtype.UUID, author HookAuthor) error { + return s.inTx(ctx, func(qtx *db.Queries) error { + if _, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author); err != nil { + return err + } + if _, err := qtx.ArchiveHook(ctx, db.ArchiveHookParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { + return err + } + return nil + }) +} + +// ListExecutions returns the newest execution-trace rows for a hook (bounded). +func (s *HookService) ListExecutions(ctx context.Context, workspaceID, hookID pgtype.UUID, limit int32) ([]db.HookExecution, error) { + // Confirm the hook belongs to the workspace before exposing its trace. + if _, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrHookNotFound + } + return nil, err + } + return s.Queries.ListHookExecutionsByHook(ctx, db.ListHookExecutionsByHookParams{HookID: hookID, Limit: limit}) +} + +// lockEditableHook loads and row-locks a non-archived, non-system hook and +// enforces the edit-authorization gate against the editor's LIVE membership and +// role read inside the same transaction. Returns the locked row for callers that +// need its stored principal. +func (s *HookService) lockEditableHook(ctx context.Context, qtx *db.Queries, workspaceID, hookID pgtype.UUID, author HookAuthor) (db.Hook, error) { + existing, err := qtx.GetHookForUpdate(ctx, db.GetHookForUpdateParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return db.Hook{}, ErrHookNotFound + } + return db.Hook{}, err + } + if existing.ArchivedAt.Valid { + return db.Hook{}, ErrHookNotFound + } + if existing.Origin == "system" { + return db.Hook{}, ErrHookSystemManaged + } + // FOR SHARE so a concurrent removal/demotion of the editor cannot commit before + // this edit does — the authorization decision stays consistent to commit. + editor, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ + UserID: author.PrincipalUserID, WorkspaceID: workspaceID, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // The editor is not (or no longer) a member of this workspace. + return db.Hook{}, ErrHookForbidden + } + return db.Hook{}, err + } + if err := authorizeHookEdit(existing, editor); err != nil { + return db.Hook{}, err + } + return existing, nil +} + +// requireLivePrincipal loads the hook's stored principal as a workspace member, +// making live membership a hard precondition for re-arming a hook (update/enable). +// A departed principal fails closed; a real DB error is propagated. +func (s *HookService) requireLivePrincipal(ctx context.Context, qtx *db.Queries, workspaceID, principal pgtype.UUID) (db.Member, error) { + // FOR SHARE so a concurrent removal of the stored principal cannot commit before + // the hook is re-armed under that principal's (now-stale) authority. + member, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ + UserID: principal, WorkspaceID: workspaceID, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return db.Member{}, ErrHookPrincipalDeparted + } + return db.Member{}, err + } + return member, nil +} + +// authorizeHookEdit implements the edit gate (review point 1): a workspace +// owner/admin may edit any hook's configuration; otherwise only the hook's +// original authorization principal may. The principal is NOT transferred on +// edit, so an arbitrary member can never rewrite a rule that keeps running under +// someone else's authority. +func authorizeHookEdit(hook db.Hook, editor db.Member) error { + if admission.RoleAllowed(editor.Role, "owner", "admin") { + return nil + } + if principalMatches(hook.AuthorizationPrincipalUserID, editor.UserID) { + return nil + } + return ErrHookForbidden +} + +func principalMatches(a, b pgtype.UUID) bool { + return a.Valid && b.Valid && a.Bytes == b.Bytes +} + +func (s *HookService) inTx(ctx context.Context, fn func(qtx *db.Queries) error) error { + return s.inTxWith(ctx, func(_ pgx.Tx, qtx *db.Queries) error { return fn(qtx) }) +} + +// inTxWith is inTx with the raw transaction exposed, for callers that need to open +// SAVEPOINTs inside it (pgx models a nested Begin as a savepoint). The matcher uses +// it to isolate one candidate hook's failure without abandoning the event's single +// authoritative transaction. +func (s *HookService) inTxWith(ctx context.Context, fn func(tx pgx.Tx, qtx *db.Queries) error) error { + tx, err := s.TxStarter.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + if err := fn(tx, s.Queries.WithTx(tx)); err != nil { + return err + } + return tx.Commit(ctx) +} + +// resolveScope maps the optional scope spec to (scope_type, scope_id). The spec +// has already passed automation.Validate, so an issue scope always has a valid id. +func resolveScope(scope *automation.ScopeSpec) (string, pgtype.UUID, error) { + if scope == nil || scope.Type == automation.ScopeWorkspace { + return automation.ScopeWorkspace, pgtype.UUID{}, nil + } + id, err := util.ParseUUID(scope.ID) + if err != nil { + return "", pgtype.UUID{}, err + } + return automation.ScopeIssue, id, nil +} + +// marshalRevisionConfig produces the JSONB payloads stored on a revision. +// conditions and actions are always stored as (possibly empty) arrays; match as +// an object. +func marshalRevisionConfig(spec automation.HookSpec) (match, conditions, actions []byte, err error) { + match = spec.When.Match + if len(match) == 0 { + match = []byte("{}") + } + if len(spec.If) == 0 { + conditions = []byte("[]") + } else if conditions, err = json.Marshal(spec.If); err != nil { + return nil, nil, nil, err + } + if actions, err = json.Marshal(spec.Do); err != nil { + return nil, nil, nil, err + } + return match, conditions, actions, nil +} + +// targetChecker fail-closed validates every id a spec references, against the +// workspace and the hook's stored principal, inside the write transaction +// (review round 3, points 1 & 2). §13 requires this at create/update time so an +// illegal configuration never enters the store and never reaches a worker. +type targetChecker struct { + ctx context.Context + qtx *db.Queries + workspaceID pgtype.UUID + principalUserID string + principalMember db.Member + principalIsMember bool +} + +// validateTargets is always called with the hook's stored principal already +// resolved to a LIVE workspace member (the create/update paths make that a hard +// gate first), so every target admission runs against a known-member principal. +func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, principal db.Member) error { + tc := &targetChecker{ + ctx: ctx, + qtx: qtx, + workspaceID: workspaceID, + principalUserID: util.UUIDToString(principal.UserID), + principalMember: principal, + principalIsMember: true, + } + return tc.validate(spec) +} + +func (tc *targetChecker) validate(spec automation.HookSpec) error { + if spec.Scope != nil && spec.Scope.Type == automation.ScopeIssue { + if err := tc.requireIssue(spec.Scope.ID, "scope.id"); err != nil { + return err + } + } + for i, cond := range spec.If { + if err := tc.validateConditionTargets(i, cond); err != nil { + return err + } + } + for i, action := range spec.Do { + if err := tc.validateActionTargets(i, action); err != nil { + return err + } + } + return nil +} + +func (tc *targetChecker) validateConditionTargets(i int, c automation.ConditionSpec) error { + if c.IssuesStatus != nil { + for _, id := range c.IssuesStatus.IDs { + if err := tc.requireIssue(id, fmt.Sprintf("if[%d].issues_status.ids", i)); err != nil { + return err + } + } + } + if c.IssueField != nil { + if err := tc.requireIssue(c.IssueField.ID, fmt.Sprintf("if[%d].issue_field.id", i)); err != nil { + return err + } + // The operand ids must also resolve to workspace resources. + operands := collectFieldOperands(*c.IssueField) + where := fmt.Sprintf("if[%d].issue_field", i) + switch c.IssueField.Field { + case automation.IssueFieldParentIssueID: + for _, v := range operands { + if err := tc.requireIssue(v, where); err != nil { + return err + } + } + case automation.IssueFieldAssigneeID: + for _, v := range operands { + if err := tc.requireAssignee(v, where); err != nil { + return err + } + } + } + } + return nil +} + +func (tc *targetChecker) validateActionTargets(i int, a automation.ActionSpec) error { + where := fmt.Sprintf("do[%d].%s", i, a.Type) + switch a.Type { + case automation.ActionSetIssueStatus, automation.ActionAddComment: + return tc.requireIssue(a.IssueID, where+".issue_id") + case automation.ActionTriggerAgent: + if err := tc.requireIssue(a.IssueID, where+".issue_id"); err != nil { + return err + } + return tc.requireInvokableAgent(a.AgentID, where+".agent_id") + case automation.ActionSendInbox: + return tc.requireMember(a.MemberID, where+".member_id") + case automation.ActionRunAutopilot: + return tc.requireWritableAutopilot(a.AutopilotID, where+".autopilot_id") + } + return nil +} + +func collectFieldOperands(c automation.IssueFieldCond) []string { + out := make([]string, 0, len(c.In)+1) + if c.Eq != "" { + out = append(out, c.Eq) + } + out = append(out, c.In...) + return out +} + +func (tc *targetChecker) requireIssue(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := tc.qtx.GetIssueInWorkspace(tc.ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references issue %s which does not exist in this workspace", field, id) + } + return err + } + return nil +} + +func (tc *targetChecker) requireMember(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := tc.qtx.GetMemberInWorkspace(tc.ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references member %s which is not in this workspace", field, id) + } + return err + } + return nil +} + +// requireAssignee validates an issue-assignee operand. Issue assignees are +// polymorphic (member | agent | squad, §Domain): a MEMBER assignee id is a USER +// id (looked up via GetMemberByUserAndWorkspace, NOT a member-row id), an agent +// id is a workspace agent, and a squad id is a non-archived workspace squad +// (review round 4, point 3). +func (tc *targetChecker) requireAssignee(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := tc.qtx.GetMemberByUserAndWorkspace(tc.ctx, db.GetMemberByUserAndWorkspaceParams{UserID: uid, WorkspaceID: tc.workspaceID}); err == nil { + return nil + } + if _, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil { + return nil + } + if squad, err := tc.qtx.GetSquadInWorkspace(tc.ctx, db.GetSquadInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil && !squad.ArchivedAt.Valid { + return nil + } + return automation.NewValidationError("%s references assignee %s which is not a member, agent, or squad in this workspace", field, id) +} + +func (tc *targetChecker) requireWritableAutopilot(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + ap, err := tc.qtx.GetAutopilotInWorkspace(tc.ctx, db.GetAutopilotInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s which does not exist in this workspace", field, id) + } + return err + } + // Write permission is judged for the hook's stored principal: role/authorship + // or an explicit collaborator grant — the same rule the interactive autopilot + // write path enforces (review round 3, point 2). + if !tc.principalIsMember { + return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + } + if !admission.AutopilotWriteByOwnership(ap, tc.principalMember) { + granted, err := tc.qtx.IsAutopilotCollaborator(tc.ctx, db.IsAutopilotCollaboratorParams{AutopilotID: ap.ID, UserID: tc.principalMember.UserID}) + if err != nil { + return err + } + if !granted { + return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + } + } + // Write permission is not enough: running the autopilot invokes its assignee + // (an agent, or a squad's leader agent), so the principal must also be able to + // invoke that agent — otherwise execution would be admission-skipped (review + // round 4, point 2). PR4 re-validates dynamic state at execution time. + return tc.requireInvocableAutopilotAssignee(ap, field) +} + +// requireInvocableAutopilotAssignee resolves the autopilot's assignee to its +// executing agent (the agent itself, or a non-archived squad's leader) and +// requires the stored principal be able to invoke it. A dangling/archived +// assignee fails closed. +func (tc *targetChecker) requireInvocableAutopilotAssignee(ap db.Autopilot, field string) error { + switch ap.AssigneeType { + case "agent": + agent, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: ap.AssigneeID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose assignee agent is missing or not in this workspace", field, util.UUIDToString(ap.ID)) + } + return err + } + return tc.checkAgentInvocable(agent, field) + case "squad": + squad, err := tc.qtx.GetSquadInWorkspace(tc.ctx, db.GetSquadInWorkspaceParams{ID: ap.AssigneeID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose squad is missing or not in this workspace", field, util.UUIDToString(ap.ID)) + } + return err + } + if squad.ArchivedAt.Valid { + return automation.NewValidationError("%s references autopilot %s whose squad is archived", field, util.UUIDToString(ap.ID)) + } + leader, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: squad.LeaderID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose squad leader is missing", field, util.UUIDToString(ap.ID)) + } + return err + } + return tc.checkAgentInvocable(leader, field) + default: + return automation.NewValidationError("%s references autopilot %s with an unsupported assignee", field, util.UUIDToString(ap.ID)) + } +} + +// requireInvokableAgent confirms the agent exists in the workspace, is not +// archived, has a runtime, and is invokable by the hook's STORED principal — the +// same admission the interactive trigger path enforces, applied fail-closed at +// save against the principal, never the editor (review round 3, point 1). +func (tc *targetChecker) requireInvokableAgent(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + agent, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references agent %s which does not exist in this workspace", field, id) + } + return err + } + return tc.checkAgentInvocable(agent, field) +} + +// checkAgentInvocable fail-closed asserts an already-loaded workspace agent is +// live (not archived, has a runtime) and invocable by the hook's stored +// principal. Shared by trigger_agent and run_autopilot assignee validation. +func (tc *targetChecker) checkAgentInvocable(agent db.Agent, field string) error { + id := util.UUIDToString(agent.ID) + if agent.ArchivedAt.Valid || !agent.RuntimeID.Valid { + return automation.NewValidationError("%s references agent %s which is archived or has no runtime", field, id) + } + targets, err := tc.qtx.ListAgentInvocationTargets(tc.ctx, agent.ID) + if err != nil { + return err + } + if !admission.AgentInvocableByMember(agent, targets, tc.principalUserID, tc.principalIsMember) { + return automation.NewValidationError("%s references agent %s which the hook's principal may not invoke", field, id) + } + return nil +} diff --git a/server/internal/service/hook_eval.go b/server/internal/service/hook_eval.go new file mode 100644 index 00000000000..5642e2cf4e4 --- /dev/null +++ b/server/internal/service/hook_eval.go @@ -0,0 +1,163 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Read-only Event Hooks debug surface (MUL-4332 PR3, decision 2A): dry-run, +// explain and the correlation query. These reuse the single automation.Evaluate +// evaluator so an explanation can never drift from real execution, and they +// perform NO action and mutate NO durable state. + +// ErrHookEventNotFound is returned when a referenced domain event does not exist +// in the workspace. +var ErrHookEventNotFound = errors.New("event not found") + +// issueStateReader is the workspace-scoped StateReader the evaluator reads +// current issue state through. +type issueStateReader struct { + q *db.Queries + workspaceID pgtype.UUID +} + +func (r *issueStateReader) IssueField(ctx context.Context, issueID, field string) (string, bool, error) { + uid, err := util.ParseUUID(issueID) + if err != nil { + return "", false, nil // a malformed id can never resolve to a workspace issue + } + issue, err := r.q.GetIssueInWorkspace(ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: r.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + return "", false, err + } + switch field { + case automation.IssueFieldStatus: + return issue.Status, true, nil + case automation.IssueFieldAssigneeID: + return util.UUIDToString(issue.AssigneeID), issue.AssigneeID.Valid, nil + case automation.IssueFieldParentIssueID: + return util.UUIDToString(issue.ParentIssueID), issue.ParentIssueID.Valid, nil + } + return "", false, nil +} + +// DryRun evaluates a candidate hook spec against a historical event, read-only. +// The spec's shape is validated (so garbage is rejected early), the event's +// `when` is matched against its historical payload, and `if` conditions read +// current workspace state. +func (s *HookService) DryRun(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, eventID pgtype.UUID) (automation.Evaluation, error) { + if err := automation.Validate(spec); err != nil { + return automation.Evaluation{}, err + } + view, err := s.loadEventView(ctx, workspaceID, eventID) + if err != nil { + return automation.Evaluation{}, err + } + rev := automation.EvalRevision{ + EventType: spec.When.Event, + Match: spec.When.Match, + Conditions: spec.If, + FireMode: spec.Fire.Mode, + } + return automation.Evaluate(ctx, view, rev, &issueStateReader{q: s.Queries, workspaceID: workspaceID}) +} + +// Explain evaluates a stored hook's revision against a historical event, +// read-only. revisionNumber == 0 explains the active revision. +func (s *HookService) Explain(ctx context.Context, workspaceID, hookID, eventID pgtype.UUID, revisionNumber int32) (automation.Evaluation, error) { + hook, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.Evaluation{}, ErrHookNotFound + } + return automation.Evaluation{}, err + } + var rawRev db.HookRevision + if revisionNumber > 0 { + rawRev, err = s.Queries.GetHookRevisionByNumber(ctx, db.GetHookRevisionByNumberParams{HookID: hookID, Revision: revisionNumber}) + } else { + rawRev, err = s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + } + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.Evaluation{}, ErrHookNotFound + } + return automation.Evaluation{}, err + } + + view, err := s.loadEventView(ctx, workspaceID, eventID) + if err != nil { + return automation.Evaluation{}, err + } + rev, err := revisionToEval(rawRev) + if err != nil { + return automation.Evaluation{}, err + } + return automation.Evaluate(ctx, view, rev, &issueStateReader{q: s.Queries, workspaceID: workspaceID}) +} + +// EventsByCorrelation returns up to limit domain events in a correlation chain +// (ordered by seq), for execution-chain debugging. The limit is enforced in the +// query, not by truncating a fully-loaded chain. +func (s *HookService) EventsByCorrelation(ctx context.Context, workspaceID, correlationID pgtype.UUID, limit int32) ([]db.DomainEvent, error) { + return s.Queries.ListDomainEventsByCorrelation(ctx, db.ListDomainEventsByCorrelationParams{ + WorkspaceID: workspaceID, CorrelationID: correlationID, Limit: limit, + }) +} + +func (s *HookService) loadEventView(ctx context.Context, workspaceID, eventID pgtype.UUID) (automation.EventView, error) { + event, err := s.Queries.GetDomainEvent(ctx, eventID) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.EventView{}, ErrHookEventNotFound + } + return automation.EventView{}, err + } + // GetDomainEvent is not workspace-scoped; enforce tenant isolation here. + if !principalMatches(event.WorkspaceID, workspaceID) { + return automation.EventView{}, ErrHookEventNotFound + } + return eventToView(event) +} + +func eventToView(e db.DomainEvent) (automation.EventView, error) { + var payload map[string]any + if len(e.Payload) > 0 { + if err := json.Unmarshal(e.Payload, &payload); err != nil { + return automation.EventView{}, err + } + } + return automation.EventView{ + Type: e.Type, + SubjectID: util.UUIDToString(e.SubjectID), + ActorType: e.ActorType, + ActorID: util.UUIDToString(e.ActorID), + Payload: payload, + }, nil +} + +func revisionToEval(rev db.HookRevision) (automation.EvalRevision, error) { + var conds []automation.ConditionSpec + if len(rev.Conditions) > 0 { + if err := json.Unmarshal(rev.Conditions, &conds); err != nil { + return automation.EvalRevision{}, err + } + } + return automation.EvalRevision{ + EventType: rev.EventType, + Match: rev.Match, + Conditions: conds, + FireMode: rev.FireMode, + }, nil +} diff --git a/server/internal/service/hook_executor.go b/server/internal/service/hook_executor.go new file mode 100644 index 00000000000..b2cd5f794be --- /dev/null +++ b/server/internal/service/hook_executor.go @@ -0,0 +1,582 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/domainevent" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The Event Hooks executor (MUL-4332 PR3 §7.2). It leases `queued` executions the +// matcher produced and runs their actions in order, resuming from +// current_action_index so a restart never re-runs work that already committed. +// +// EVERY action is anchored by a durable effect row keyed on (execution, action +// index). For a platform DB action the effect, the target write and the resulting +// domain event all commit in ONE transaction (§4), so the crash window between +// "action happened" and "we recorded that it happened" does not exist: either all +// three are durable or none are. A retry that finds a succeeded effect skips the +// action and reuses its recorded output rather than repeating it. +// +// OWNERSHIP is one predicate — right token, still `running`, and not expired under +// DATABASE clock time — asserted before any action write and re-applied to every +// status write. A worker whose lease was reclaimed, or whose own lease elapsed +// mid-action, commits nothing and can never write terminal state (§7.3). +// +// Actions are NOT collectively atomic, by design: action 1 succeeding and action 2 +// finally failing is an explicit partial execution and action 1 is not rolled back +// (§7.2). The action cursor advances in the same transaction as the action, so the +// retry resumes precisely at the action that failed. +// +// This slice implements `set_issue_status`. `trigger_agent` needs the task enqueue +// path to become transaction-aware first, so its effect can bind in the same +// transaction as the enqueue (§4: "task / comment / inbox 可在同一事务内绑定 +// effect"); until then it is left unimplemented rather than run without that +// guarantee. The executor loop is gated on the default-off automation_event_hooks +// flag, so production behaviour is unchanged. + +const ( + hookExecRunning = "running" + hookExecSucceeded = "succeeded" + + // Terminal, non-retryable skip reasons (§7.3). + skipTargetUnavailable = "target_unavailable" + skipPrincipalInvalid = "principal_invalid" + skipActionUnsupported = "action_unsupported" + + // errCodeInfra marks an execution that exhausted its infrastructure retries. + errCodeInfra = "infra" + + // hookDisabledPrincipalInvalid is recorded on a hook paused because its stored + // authorization principal is no longer a workspace member. + hookDisabledPrincipalInvalid = "principal_invalid" + + // ExecutorBatchSize bounds how many executions one executor tick runs. + ExecutorBatchSize = 20 +) + +// ExecutorLeaseTTL bounds how long one execution may hold its lease. It is a var so +// tests can drive the expired-lease path deterministically. +var ExecutorLeaseTTL = 2 * time.Minute + +// executorBackoff is the infrastructure-failure retry ladder (§7.3). An execution +// that exhausts it is marked failed. +var executorBackoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute} + +// errExecutionLeaseLost means this worker no longer owns the execution — reclaimed, +// already finalized, or its own lease expired. It always aborts the action +// transaction, so a non-owner commits nothing and writes no terminal state. +var errExecutionLeaseLost = errors.New("hook executor: execution lease lost") + +// actionSkip is a terminal, non-retryable action outcome: the rule cannot run as +// written and retrying would fail identically (§7.3). It finalizes the execution as +// skipped rather than consuming the infrastructure retry ladder. +type actionSkip struct { + reason string + detail string +} + +func (e *actionSkip) Error() string { return e.reason + ": " + e.detail } + +func skipAction(reason, format string, args ...any) *actionSkip { + return &actionSkip{reason: reason, detail: fmt.Sprintf(format, args...)} +} + +// effectKeyFor is the durable idempotency key of one action of one execution. +func effectKeyFor(executionID pgtype.UUID, index int) string { + return fmt.Sprintf("%s:%d", util.UUIDToString(executionID), index) +} + +// ClaimAndRun leases and runs up to batchSize executions, returning how many reached +// a terminal state in this tick. +func (s *HookService) ClaimAndRun(ctx context.Context, batchSize int32) (int, error) { + finished := 0 + for i := int32(0); i < batchSize; i++ { + claimed, done, err := s.claimAndRunOne(ctx) + if err != nil { + slog.Warn("hook executor: execution failed", "error", err) + return finished, nil + } + if !claimed { + break // queue drained + } + if done { + finished++ + } + } + return finished, nil +} + +// claimAndRunOne leases one execution and runs its remaining actions. It reports +// whether an execution was claimed and whether it reached a terminal state. +func (s *HookService) claimAndRunOne(ctx context.Context) (claimed bool, finished bool, err error) { + lease := util.NewUUID() + exec, err := s.Queries.ClaimOneHookExecution(ctx, db.ClaimOneHookExecutionParams{ + LeaseToken: lease, + LeaseTtlSeconds: ExecutorLeaseTTL.Seconds(), + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, false, nil + } + return false, false, err + } + + actions, err := s.executionActions(ctx, exec) + if err != nil { + // The revision cannot be read or parsed: deterministic, so retrying is + // pointless. Record it terminally instead of burning the retry ladder. + return true, s.finalizeSkippedAction(ctx, exec, lease, automation.ActionSpec{}, + int(exec.CurrentActionIndex), skipAction(skipActionUnsupported, "%v", err)), nil + } + + for idx := int(exec.CurrentActionIndex); idx < len(actions); idx++ { + // Renew before each action so a live worker running a long chain keeps its + // claim, while an ABANDONED claim still expires on schedule. + if _, err := s.Queries.HeartbeatHookExecution(ctx, db.HeartbeatHookExecutionParams{ + ID: exec.ID, LeaseToken: lease, LeaseTtlSeconds: ExecutorLeaseTTL.Seconds(), + }); err != nil { + return true, s.rescheduleOrFail(ctx, exec, lease, actions[idx], idx, err), nil + } + + runErr := s.runAction(ctx, exec, lease, actions[idx], idx) + if runErr == nil { + continue + } + var skip *actionSkip + switch { + case errors.Is(runErr, errExecutionLeaseLost): + // We wrote nothing. If the row still carries OUR token the lease simply + // elapsed under us, so back it off: leaving it untouched keeps it at the + // head of the queue, where the next claim selects it again immediately and + // starves everything behind it. If another worker has reclaimed it the + // token differs, nothing is written, and the new owner is left alone. + return true, false, s.deferExpiredExecution(ctx, exec, lease) + case errors.As(runErr, &skip): + return true, s.finalizeSkippedAction(ctx, exec, lease, actions[idx], idx, skip), nil + default: + // Infrastructure failure: back off and resume at THIS action. Every + // action already committed stays committed. + return true, s.rescheduleOrFail(ctx, exec, lease, actions[idx], idx, runErr), nil + } + } + + rows, err := s.Queries.MarkHookExecutionSucceeded(ctx, db.MarkHookExecutionSucceededParams{ + ID: exec.ID, LeaseToken: lease, + }) + if err != nil { + return true, false, err + } + return true, rows == 1, nil +} + +// executionActions loads the actions of the revision this execution was pinned to at +// match time. A later edit to the hook never changes what a created execution runs. +func (s *HookService) executionActions(ctx context.Context, exec db.HookExecution) ([]automation.ActionSpec, error) { + rev, err := s.Queries.GetHookRevision(ctx, exec.HookRevisionID) + if err != nil { + return nil, err + } + var actions []automation.ActionSpec + if len(rev.Actions) > 0 { + if err := json.Unmarshal(rev.Actions, &actions); err != nil { + return nil, fmt.Errorf("%w: parse stored actions: %v", automation.ErrInvalidConfig, err) + } + } + return actions, nil +} + +// runAction performs one action, records its effect, and advances the cursor — all +// in a single transaction, so the action and the record that it happened are never +// separable. +func (s *HookService) runAction(ctx context.Context, exec db.HookExecution, lease pgtype.UUID, action automation.ActionSpec, index int) error { + effectKey := effectKeyFor(exec.ID, index) + + return domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + // Ownership, fail-closed, before any write. + owned, err := qtx.GetOwnedHookExecution(ctx, db.GetOwnedHookExecutionParams{ID: exec.ID, LeaseToken: lease}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, errExecutionLeaseLost + } + return nil, err + } + + // An effect that already succeeded means this action is durably done; skip it + // and just carry the cursor forward. + existing, err := qtx.GetHookActionEffect(ctx, effectKey) + switch { + case err == nil && existing.Status == hookExecSucceeded: + return nil, advanceCursor(ctx, qtx, owned, lease, index) + case err != nil && !errors.Is(err, pgx.ErrNoRows): + return nil, err + } + // A non-succeeded anchor is a previous terminal attempt (skipped/failed). This + // retry re-attempts the action and the write below updates that same row. + retryOfTerminal := err == nil + + // The principal's authority is re-checked for EVERY action, against live + // membership, not the snapshot taken when the hook was saved (§8). + if err := s.requireExecutionPrincipal(ctx, qtx, owned); err != nil { + return nil, err + } + + resolved, err := json.Marshal(action) + if err != nil { + return nil, err + } + // Claim the anchor. Losing the race means a concurrent attempt owns this + // action; ownership above makes that a lost lease, so stop rather than run it + // a second time. + if _, err := qtx.CreateHookActionEffect(ctx, db.CreateHookActionEffectParams{ + ID: util.NewUUID(), + EffectKey: effectKey, + ExecutionID: owned.ID, + ActionIndex: int32(index), + ActionType: action.Type, + ResolvedInput: resolved, + }); err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return nil, err + } + // ON CONFLICT DO NOTHING returned nothing. If we already saw a terminal + // anchor this is our own retry and we continue; otherwise the anchor + // appeared between our read and our write, so a concurrent attempt owns + // this action and we must not run it a second time. + if !retryOfTerminal { + return nil, errExecutionLeaseLost + } + } + + events, outputType, outputID, err := s.performAction(ctx, qtx, owned, action, index) + if err != nil { + return nil, err + } + + if _, err := qtx.MarkHookActionEffectSucceeded(ctx, db.MarkHookActionEffectSucceededParams{ + EffectKey: effectKey, + OutputType: pgtype.Text{String: outputType, Valid: outputType != ""}, + OutputID: outputID, + }); err != nil { + return nil, err + } + if err := advanceCursor(ctx, qtx, owned, lease, index); err != nil { + return nil, err + } + return events, nil + }) +} + +func advanceCursor(ctx context.Context, qtx *db.Queries, exec db.HookExecution, lease pgtype.UUID, index int) error { + rows, err := qtx.AdvanceHookExecutionAction(ctx, db.AdvanceHookExecutionActionParams{ + ID: exec.ID, LeaseToken: lease, NextActionIndex: int32(index + 1), + }) + if err != nil { + return err + } + if rows != 1 { + return errExecutionLeaseLost + } + return nil +} + +// requireExecutionPrincipal re-asserts that the hook's stored authorization +// principal is still a workspace member. A departed principal is terminal: the hook +// is paused so it stops producing work under authority nobody holds any more (§7.3). +func (s *HookService) requireExecutionPrincipal(ctx context.Context, qtx *db.Queries, exec db.HookExecution) error { + hook, err := qtx.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: exec.HookID, WorkspaceID: exec.WorkspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return skipAction(skipTargetUnavailable, "hook %s no longer exists", util.UUIDToString(exec.HookID)) + } + return err + } + if !hook.AuthorizationPrincipalUserID.Valid { + return skipAction(skipPrincipalInvalid, "hook %s has no authorization principal", util.UUIDToString(hook.ID)) + } + if _, err := s.requireLivePrincipal(ctx, qtx, exec.WorkspaceID, hook.AuthorizationPrincipalUserID); err != nil { + if errors.Is(err, ErrHookPrincipalDeparted) { + // The pause itself is applied by the caller, AFTER this transaction rolls + // back: writing it here would be undone by the very error that reports it. + return skipAction(skipPrincipalInvalid, + "hook %s authorization principal is no longer a workspace member", util.UUIDToString(hook.ID)) + } + return err + } + return nil +} + +// pauseHookForInvalidPrincipal disables a hook whose stored authorization principal +// has left the workspace, so it stops producing work under authority nobody holds any +// more (§7.3), and notifies the workspace owners/admins who can act on it. It runs in +// the CALLER's transaction: the pause must be durable in the same commit as the +// terminal skip, or the rule would keep firing while the execution reads as handled. +func (s *HookService) pauseHookForInvalidPrincipal(ctx context.Context, qtx *db.Queries, exec db.HookExecution) error { + hook, err := qtx.SetHookEnabled(ctx, db.SetHookEnabledParams{ + ID: exec.HookID, WorkspaceID: exec.WorkspaceID, Enabled: false, + DisabledReason: pgtype.Text{String: hookDisabledPrincipalInvalid, Valid: true}, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil // already archived; nothing left to pause + } + return err + } + return s.notifyAdminsHookPaused(ctx, qtx, hook) +} + +// notifyAdminsHookPaused tells the people who can re-arm the rule that it stopped. +// A paused hook is silent by nature, so without this the rule simply stops working +// with nothing surfaced to anyone (§7.3). +func (s *HookService) notifyAdminsHookPaused(ctx context.Context, qtx *db.Queries, hook db.Hook) error { + admins, err := qtx.ListWorkspaceMembersByRoles(ctx, db.ListWorkspaceMembersByRolesParams{ + WorkspaceID: hook.WorkspaceID, Roles: []string{"owner", "admin"}, + }) + if err != nil { + return err + } + details, err := json.Marshal(map[string]any{ + "hook_id": util.UUIDToString(hook.ID), + "reason": hookDisabledPrincipalInvalid, + }) + if err != nil { + return err + } + for _, admin := range admins { + if _, err := qtx.CreateInboxItem(ctx, db.CreateInboxItemParams{ + WorkspaceID: hook.WorkspaceID, + RecipientType: "member", + RecipientID: admin.UserID, + Type: "hook_paused", + Severity: "action_required", + Title: "Automation paused: " + hook.Name, + Body: pgtype.Text{String: "Its authorization principal is no longer a workspace member, so the rule was disabled.", Valid: true}, + ActorType: pgtype.Text{String: "system", Valid: true}, + Details: details, + }); err != nil { + return err + } + } + return nil +} + +// performAction dispatches one action. It returns the domain events the action +// produced, which the caller commits in the same transaction, plus the effect output. +func (s *HookService) performAction(ctx context.Context, qtx *db.Queries, exec db.HookExecution, action automation.ActionSpec, index int) ([]domainevent.Event, string, pgtype.UUID, error) { + switch action.Type { + case automation.ActionSetIssueStatus: + return s.actionSetIssueStatus(ctx, qtx, exec, action, index) + default: + // Reached only for an action this slice does not implement yet. Terminal + // rather than retried, so it cannot loop on the backoff ladder. + return nil, "", pgtype.UUID{}, skipAction(skipActionUnsupported, + "action %q is not implemented by this executor slice", action.Type) + } +} + +// actionSetIssueStatus writes the target status and emits the resulting +// issue.status_changed event in the caller's transaction, so the fact and its event +// commit together exactly as every other domain write does. +func (s *HookService) actionSetIssueStatus(ctx context.Context, qtx *db.Queries, exec db.HookExecution, action automation.ActionSpec, index int) ([]domainevent.Event, string, pgtype.UUID, error) { + issueID, err := util.ParseUUID(action.IssueID) + if err != nil { + return nil, "", pgtype.UUID{}, skipAction(skipTargetUnavailable, "issue_id %q is not a uuid", action.IssueID) + } + // Workspace-scoped, so an action can never reach across tenants (§8). + before, err := qtx.LockIssueRowForUpdate(ctx, db.LockIssueRowForUpdateParams{ID: issueID, WorkspaceID: exec.WorkspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", pgtype.UUID{}, skipAction(skipTargetUnavailable, + "issue %s is not in this workspace", action.IssueID) + } + return nil, "", pgtype.UUID{}, err + } + updated, err := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ + ID: issueID, WorkspaceID: exec.WorkspaceID, Status: action.Status, + }) + if err != nil { + return nil, "", pgtype.UUID{}, err + } + // A no-op transition is a successful action that emits nothing, matching every + // other status write in the codebase. + if before.Status == updated.Status { + return nil, "issue", issueID, nil + } + + evt := domainevent.IssueStatusChanged(exec.WorkspaceID, issueID, + domainevent.ActorFrom("hook", exec.HookID), + domainevent.IssueStatusChangedPayload{From: before.Status, To: updated.Status}) + // The reaction stays in its originating chain and records what caused it, so the + // depth guard can see how deep this chain has run. + evt.CorrelationID = exec.CorrelationID + evt.CausationExecutionID = exec.ID + evt.CausationActionIndex = pgtype.Int4{Int32: int32(index), Valid: true} + hop, err := s.sourceHopCount(ctx, qtx, exec.EventID) + if err != nil { + return nil, "", pgtype.UUID{}, err + } + evt.HopCount = hop + 1 + + return []domainevent.Event{evt}, "issue", issueID, nil +} + +// sourceHopCount reads the depth of the event that produced this execution, so the +// event this action emits sits one hop deeper in the same causal chain. +func (s *HookService) sourceHopCount(ctx context.Context, qtx *db.Queries, eventID pgtype.UUID) (int32, error) { + src, err := qtx.GetDomainEvent(ctx, eventID) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return 0, nil // source aged out of retention; treat as a root + } + return 0, err + } + return src.HopCount, nil +} + +// finalizeSkippedAction commits a terminal, non-retryable outcome: the action's +// durable effect row and the execution's skipped status, atomically. It reports +// whether this worker still owned the execution and therefore actually finalized it. +// +// A departed principal additionally pauses the hook, and that pause must be DURABLE +// before the skip is committed. Recording the skip while the hook stays enabled would +// be fail-open: the rule would keep producing executions under authority nobody +// holds. So the pause joins the same transaction, and if it cannot be written the +// whole thing is treated as a retryable infrastructure failure instead. +func (s *HookService) finalizeSkippedAction(ctx context.Context, exec db.HookExecution, lease pgtype.UUID, action automation.ActionSpec, index int, skip *actionSkip) bool { + slog.Warn("hook executor: action skipped", "execution_id", util.UUIDToString(exec.ID), + "action_index", index, "reason", skip.reason, "detail", skip.detail) + + err := s.inTx(ctx, func(qtx *db.Queries) error { + if skip.reason == skipPrincipalInvalid { + if err := s.pauseHookForInvalidPrincipal(ctx, qtx, exec); err != nil { + return err + } + } + if err := s.writeTerminalEffect(ctx, qtx, exec, action, index, hookExecSkipped, skip.reason, skip.detail); err != nil { + return err + } + rows, err := qtx.MarkHookExecutionSkipped(ctx, db.MarkHookExecutionSkippedParams{ + ID: exec.ID, LeaseToken: lease, SkipReason: pgtype.Text{String: skip.reason, Valid: true}, + }) + if err != nil { + return err + } + if rows != 1 { + return errExecutionLeaseLost + } + return nil + }) + switch { + case err == nil: + return true + case errors.Is(err, errExecutionLeaseLost): + return false + default: + // The terminal record could not be written. Do NOT leave the execution + // half-finalized: re-queue it so the outcome is recorded on a later attempt. + slog.Warn("hook executor: could not commit terminal skip, retrying", + "execution_id", util.UUIDToString(exec.ID), "reason", skip.reason, "error", err) + s.rescheduleOrFail(ctx, exec, lease, action, index, err) + return false + } +} + +// writeTerminalEffect records the durable audit row for an action that ended +// terminally. The success path writes its effect inside the action transaction, +// which rolls back on failure, so without this a skipped or failed action would +// leave no trace and a partial execution would show only the actions that succeeded. +func (s *HookService) writeTerminalEffect(ctx context.Context, qtx *db.Queries, exec db.HookExecution, action automation.ActionSpec, index int, status, code, detail string) error { + resolved, err := json.Marshal(action) + if err != nil { + return err + } + return qtx.UpsertTerminalHookActionEffect(ctx, db.UpsertTerminalHookActionEffectParams{ + ID: util.NewUUID(), + EffectKey: effectKeyFor(exec.ID, index), + ExecutionID: exec.ID, + ActionIndex: int32(index), + ActionType: action.Type, + Status: status, + ResolvedInput: resolved, + ErrorCode: pgtype.Text{String: code, Valid: code != ""}, + Error: pgtype.Text{String: detail, Valid: detail != ""}, + }) +} + +// deferExpiredExecution backs off an execution whose lease elapsed while this worker +// was running it, so it cannot hold the head of the queue. The CAS fires only while +// the row still carries our token, so a row another worker has already reclaimed is +// left untouched. +func (s *HookService) deferExpiredExecution(ctx context.Context, exec db.HookExecution, lease pgtype.UUID) error { + backoff := executorBackoff[0] + rows, err := s.Queries.DeferExpiredHookExecution(ctx, db.DeferExpiredHookExecutionParams{ + ID: exec.ID, LeaseToken: lease, BackoffSeconds: int32(backoff.Seconds()), + }) + if err != nil { + return err + } + if rows == 1 { + slog.Warn("hook executor: lease expired mid-execution, backed off", + "execution_id", util.UUIDToString(exec.ID), "backoff", backoff) + } + return nil +} + +// rescheduleOrFail applies the infrastructure retry ladder, or marks the execution +// failed once it is exhausted. It reports whether the execution reached a terminal +// state. +func (s *HookService) rescheduleOrFail(ctx context.Context, exec db.HookExecution, lease pgtype.UUID, action automation.ActionSpec, index int, cause error) bool { + // attempts was incremented by the claim, so attempt N reads as attempts == N. + if attempt := int(exec.Attempts); attempt <= len(executorBackoff) { + backoff := executorBackoff[attempt-1] + rows, err := s.Queries.RescheduleHookExecution(ctx, db.RescheduleHookExecutionParams{ + ID: exec.ID, LeaseToken: lease, BackoffSeconds: int32(backoff.Seconds()), + }) + if err != nil { + slog.Warn("hook executor: could not reschedule execution", + "execution_id", util.UUIDToString(exec.ID), "error", err) + return false + } + if rows == 1 { + slog.Warn("hook executor: action failed, retrying after backoff", + "execution_id", util.UUIDToString(exec.ID), "attempt", attempt, + "backoff", backoff, "error", cause) + } + return false // re-queued, not terminal + } + + // The ladder is spent. Record the action's terminal effect and the execution's + // failure together, so an exhausted action leaves the same audit trail a + // successful one does. + var finalized bool + if err := s.inTx(ctx, func(qtx *db.Queries) error { + if err := s.writeTerminalEffect(ctx, qtx, exec, action, index, hookExecFailed, errCodeInfra, cause.Error()); err != nil { + return err + } + rows, err := qtx.MarkHookExecutionFailed(ctx, db.MarkHookExecutionFailedParams{ + ID: exec.ID, LeaseToken: lease, + ErrorCode: pgtype.Text{String: errCodeInfra, Valid: true}, + Error: pgtype.Text{String: cause.Error(), Valid: true}, + }) + if err != nil { + return err + } + finalized = rows == 1 + return nil + }); err != nil { + slog.Warn("hook executor: could not mark execution failed", + "execution_id", util.UUIDToString(exec.ID), "error", err) + return false + } + return finalized +} diff --git a/server/internal/service/hook_executor_test.go b/server/internal/service/hook_executor_test.go new file mode 100644 index 00000000000..a34f635f04e --- /dev/null +++ b/server/internal/service/hook_executor_test.go @@ -0,0 +1,723 @@ +package service + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// seedQueuedExecution creates the hook + revision + source event + `queued` +// execution the matcher would have produced, so the executor has something to claim. +func (f matcherFixture) seedQueuedExecution(t *testing.T, actionsJSON string) (hookID, execID string, event db.DomainEvent) { + t.Helper() + ctx := context.Background() + hookID, revID := uuid.NewString(), uuid.NewString() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook (id, workspace_id, name, enabled, active_revision_id, scope_type, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id) + VALUES ($1, $2, 'x hook', true, $3, 'workspace', 'user', 'member', $4, $4)`, + hookID, f.ws, revID, f.userID); err != nil { + t.Fatalf("seed hook: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook_revision (id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id) + VALUES ($1, $2, 1, 'issue.status_changed', '{}'::jsonb, '[]'::jsonb, $3, $4::jsonb, 'member', $5)`, + revID, hookID, automation.FirePerEvent, actionsJSON, f.userID); err != nil { + t.Fatalf("seed revision: %v", err) + } + event = f.seedEvent(t, "done", 0) + execID = uuid.NewString() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook_execution (id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status) + VALUES ($1, $2, $3, $4, $5, $6, 'queued')`, + execID, f.ws, hookID, revID, util.UUIDToString(event.ID), util.UUIDToString(event.CorrelationID)); err != nil { + t.Fatalf("seed execution: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, `DELETE FROM hook_action_effect WHERE execution_id = $1`, execID) + f.pool.Exec(bg, `DELETE FROM hook_execution WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM hook_revision WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM hook WHERE id = $1`, hookID) + }) + return hookID, execID, event +} + +type execState struct { + status string + skipReason string + errorCode string + actionIndex int + attempts int + lease string + retryQueued bool +} + +func (f matcherFixture) execState(t *testing.T, execID string) execState { + t.Helper() + var s execState + if err := f.pool.QueryRow(context.Background(), ` + SELECT status, COALESCE(skip_reason,''), COALESCE(error_code,''), current_action_index, attempts, + COALESCE(lease_token::text,''), COALESCE(next_attempt_at > now(), false) + FROM hook_execution WHERE id = $1`, execID). + Scan(&s.status, &s.skipReason, &s.errorCode, &s.actionIndex, &s.attempts, &s.lease, &s.retryQueued); err != nil { + t.Fatalf("load execution state: %v", err) + } + return s +} + +func (f matcherFixture) issueStatus(t *testing.T) string { + t.Helper() + var status string + if err := f.pool.QueryRow(context.Background(), + `SELECT status FROM issue WHERE id = $1`, f.issueID).Scan(&status); err != nil { + t.Fatalf("read issue status: %v", err) + } + return status +} + +func (f matcherFixture) effectCount(t *testing.T, execID string) int { + t.Helper() + var n int + if err := f.pool.QueryRow(context.Background(), + `SELECT count(*) FROM hook_action_effect WHERE execution_id = $1`, execID).Scan(&n); err != nil { + t.Fatalf("count effects: %v", err) + } + return n +} + +// effectRows returns each action's durable effect as (action_index, status). +func (f matcherFixture) effectRows(t *testing.T, execID string) map[int]string { + t.Helper() + rows, err := f.pool.Query(context.Background(), + `SELECT action_index, status FROM hook_action_effect WHERE execution_id = $1`, execID) + if err != nil { + t.Fatalf("load effects: %v", err) + } + defer rows.Close() + out := map[int]string{} + for rows.Next() { + var idx int + var status string + if err := rows.Scan(&idx, &status); err != nil { + t.Fatalf("scan effect: %v", err) + } + out[idx] = status + } + return out +} + +func (f matcherFixture) setIssueStatusAction(status string) string { + return `[{"type":"set_issue_status","issue_id":"` + f.issueID + `","status":"` + status + `"}]` +} + +// The happy path: the action writes the target, emits the causal event, records its +// effect, and the execution reaches succeeded. +func TestExecutorRunsSetIssueStatus(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + hookID, execID, event := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("claim and run: %v", err) + } + + if got := f.issueStatus(t); got != "done" { + t.Errorf("issue status = %q, want done", got) + } + s := f.execState(t, execID) + if s.status != hookExecSucceeded { + t.Errorf("execution status = %q (skip=%q err=%q), want succeeded", s.status, s.skipReason, s.errorCode) + } + if s.actionIndex != 1 { + t.Errorf("action cursor = %d, want 1", s.actionIndex) + } + if s.lease != "" { + t.Errorf("terminal execution still holds lease %s", s.lease) + } + if n := f.effectCount(t, execID); n != 1 { + t.Errorf("effect rows = %d, want 1", n) + } + + // The emitted event stays in the originating chain, records what caused it, and + // sits one hop deeper so the depth guard can see the chain grow. + var hop int + var corr, causeExec string + var causeIdx int + if err := f.pool.QueryRow(ctx, ` + SELECT hop_count, correlation_id::text, causation_execution_id::text, causation_action_index + FROM domain_event + WHERE causation_execution_id = $1`, execID).Scan(&hop, &corr, &causeExec, &causeIdx); err != nil { + t.Fatalf("load emitted event: %v", err) + } + if hop != int(event.HopCount)+1 { + t.Errorf("emitted hop_count = %d, want %d", hop, event.HopCount+1) + } + if corr != util.UUIDToString(event.CorrelationID) { + t.Errorf("emitted correlation = %s, want inherited %s", corr, util.UUIDToString(event.CorrelationID)) + } + if causeExec != execID || causeIdx != 0 { + t.Errorf("causation = (%s, %d), want (%s, 0)", causeExec, causeIdx, execID) + } + _ = hookID +} + +// Re-running an execution whose action already committed must not repeat the action. +// This is the crash window "action succeeded but the execution cursor was not yet +// updated": the effect row is the durable anchor that closes it. +func TestExecutorEffectKeyMakesActionIdempotent(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + _, execID, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("first run: %v", err) + } + if n := f.effectCount(t, execID); n != 1 { + t.Fatalf("effect rows after first run = %d, want 1", n) + } + + // Simulate the crash window: the action committed, but the execution was left + // running with its cursor rewound, exactly as a killed worker would leave it. + if _, err := f.pool.Exec(ctx, ` + UPDATE hook_execution + SET status = 'queued', current_action_index = 0, completed_at = NULL, + lease_token = NULL, lease_expires_at = NULL + WHERE id = $1`, execID); err != nil { + t.Fatalf("rewind execution: %v", err) + } + // Move the issue away so a repeated action would be visible. + f.setIssueStatus(t, "todo") + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("second run: %v", err) + } + + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — the action ran a second time instead of "+ + "being skipped by its succeeded effect", got) + } + if n := f.effectCount(t, execID); n != 1 { + t.Errorf("effect rows = %d, want 1 (the anchor must not be duplicated)", n) + } + if s := f.execState(t, execID); s.status != hookExecSucceeded { + t.Errorf("execution status = %q, want succeeded", s.status) + } + // Exactly one causal event, from the single real execution of the action. + var events int + if err := f.pool.QueryRow(ctx, + `SELECT count(*) FROM domain_event WHERE causation_execution_id = $1`, execID).Scan(&events); err != nil { + t.Fatal(err) + } + if events != 1 { + t.Errorf("emitted %d causal events, want exactly 1", events) + } +} + +// A worker whose lease has expired must not write the target, the effect, or any +// terminal status (§7.3). +func TestExecutorExpiredLeaseWritesNothing(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + _, execID, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + original := ExecutorLeaseTTL + t.Cleanup(func() { ExecutorLeaseTTL = original }) + + // Every lease this tick acquires is already expired when granted. Ownership must + // be asserted BEFORE any action work, not merely discovered at the closing CAS: + // hold the target row, and a worker that had already started acting would block + // here. Returning promptly is the observable proof that it failed closed up front. + // This ordering is what will keep a future non-transactional action (an agent + // enqueue) from doing work a rollback cannot undo. + ExecutorLeaseTTL = -1 * time.Minute + lockConn, err := f.pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM issue WHERE id = $1 FOR UPDATE`, f.issueID); err != nil { + t.Fatalf("lock issue: %v", err) + } + + done := make(chan error, 1) + go func() { + _, err := f.svc.ClaimAndRun(ctx, 5) + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatalf("run with an expired lease: %v", err) + } + case <-time.After(1 * time.Second): + lockTx.Rollback(ctx) + lockConn.Release() + t.Fatal("expired-lease worker blocked on the target row — it began acting before asserting ownership") + } + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release issue lock: %v", err) + } + lockConn.Release() + + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — an expired-lease worker wrote the target", got) + } + if n := f.effectCount(t, execID); n != 0 { + t.Errorf("expired-lease worker wrote %d effect(s), want 0", n) + } + s := f.execState(t, execID) + if s.status == hookExecSucceeded || s.status == hookExecFailed || s.status == hookExecSkipped { + t.Errorf("expired-lease worker wrote terminal status %q", s.status) + } + + // Losing the lease now also backs the execution off (see + // TestExecutorExpiredLeaseDoesNotStarveQueue), so it is not immediately + // claimable. Advance past that backoff to reach the retry. + if _, err := f.pool.Exec(ctx, + `UPDATE hook_execution SET next_attempt_at = now() WHERE id = $1`, execID); err != nil { + t.Fatalf("clear backoff: %v", err) + } + + // With a live lease the same execution runs normally. + ExecutorLeaseTTL = original + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("run with a live lease: %v", err) + } + if got := f.issueStatus(t); got != "done" { + t.Errorf("issue status = %q, want done after a valid retry", got) + } + if s := f.execState(t, execID); s.status != hookExecSucceeded { + t.Errorf("execution status = %q, want succeeded", s.status) + } +} + +// A departed authorization principal is terminal, not retried, and pauses the hook so +// it stops producing work under authority nobody holds any more (§7.3). +func TestExecutorDepartedPrincipalSkipsAndPausesHook(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + hookID, execID, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + // The principal leaves the workspace between matching and execution. + if _, err := f.pool.Exec(ctx, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, f.ws, f.userID); err != nil { + t.Fatalf("remove principal: %v", err) + } + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("claim and run: %v", err) + } + + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — the action ran under a departed principal", got) + } + s := f.execState(t, execID) + if s.status != hookExecSkipped || s.skipReason != skipPrincipalInvalid { + t.Errorf("execution = (%q, %q), want skipped/%s", s.status, s.skipReason, skipPrincipalInvalid) + } + if s.retryQueued { + t.Error("a departed principal must be terminal, not scheduled for retry") + } + + var enabled bool + var reason string + if err := f.pool.QueryRow(ctx, + `SELECT enabled, COALESCE(disabled_reason,'') FROM hook WHERE id = $1`, hookID).Scan(&enabled, &reason); err != nil { + t.Fatal(err) + } + if enabled { + t.Error("hook was left enabled under a departed principal") + } + if reason != hookDisabledPrincipalInvalid { + t.Errorf("disabled_reason = %q, want %s", reason, hookDisabledPrincipalInvalid) + } +} + +// An action naming a target outside the workspace is terminal, not retried, and the +// tenant boundary holds. +func TestExecutorForeignTargetIsTerminalSkip(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + foreign := uuid.NewString() + actions := `[{"type":"set_issue_status","issue_id":"` + foreign + `","status":"done"}]` + _, execID, _ := f.seedQueuedExecution(t, actions) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("claim and run: %v", err) + } + + s := f.execState(t, execID) + if s.status != hookExecSkipped || s.skipReason != skipTargetUnavailable { + t.Errorf("execution = (%q, %q), want skipped/%s", s.status, s.skipReason, skipTargetUnavailable) + } + if s.retryQueued { + t.Error("an unavailable target must be terminal, not scheduled for retry") + } + // A terminal action still leaves its durable audit row, carrying the reason — + // otherwise skipped actions are invisible in the effect trace. + effects := f.effectRows(t, execID) + if len(effects) != 1 || effects[0] != hookExecSkipped { + t.Errorf("effects = %v, want exactly action 0 recorded as skipped", effects) + } + var code, resolved string + if err := f.pool.QueryRow(ctx, ` + SELECT COALESCE(error_code,''), COALESCE(resolved_input::text,'') + FROM hook_action_effect WHERE execution_id = $1`, execID).Scan(&code, &resolved); err != nil { + t.Fatal(err) + } + if code != skipTargetUnavailable { + t.Errorf("effect error_code = %q, want %s", code, skipTargetUnavailable) + } + if resolved == "" { + t.Error("terminal effect recorded no resolved_input") + } +} + +// A partial execution is explicit: action 1 committed, action 2 fails terminally, and +// action 1 is NOT rolled back (§7.2). +func TestExecutorPartialExecutionKeepsCommittedAction(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + foreign := uuid.NewString() + actions := `[{"type":"set_issue_status","issue_id":"` + f.issueID + `","status":"done"},` + + `{"type":"set_issue_status","issue_id":"` + foreign + `","status":"done"}]` + _, execID, _ := f.seedQueuedExecution(t, actions) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("claim and run: %v", err) + } + + if got := f.issueStatus(t); got != "done" { + t.Errorf("issue status = %q, want done — action 1 must not be rolled back when action 2 fails", got) + } + s := f.execState(t, execID) + if s.status != hookExecSkipped || s.skipReason != skipTargetUnavailable { + t.Errorf("execution = (%q, %q), want skipped/%s", s.status, s.skipReason, skipTargetUnavailable) + } + if s.actionIndex != 1 { + t.Errorf("action cursor = %d, want 1 (action 0 committed, action 1 failed)", s.actionIndex) + } + // The partial trace is complete: the committed action AND the one that ended it. + effects := f.effectRows(t, execID) + if len(effects) != 2 || effects[0] != hookExecSucceeded || effects[1] != hookExecSkipped { + t.Errorf("effects = %v, want action 0 succeeded and action 1 skipped — a partial "+ + "execution must show the action that failed, not only the ones that worked", effects) + } +} + +// The infrastructure retry ladder: a transient failure re-queues with backoff and +// resumes at the same action, and the execution only fails once the ladder is spent. +func TestExecutorInfraFailureBacksOffThenFails(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + _, execID, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + // Make the target write fail for real, the way an infrastructure fault would. + trigger := "exec_fail_" + util.UUIDToString(util.NewUUID())[:8] + fn := trigger + "_fn" + if _, err := f.pool.Exec(ctx, ` + CREATE FUNCTION `+quoteIdent(fn)+`() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected infrastructure failure'; END; $$;`); err != nil { + t.Fatalf("create failure function: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + CREATE TRIGGER `+quoteIdent(trigger)+` BEFORE UPDATE OF status ON issue + FOR EACH ROW WHEN (NEW.id = `+quoteLiteral(f.issueID)+`::uuid) + EXECUTE FUNCTION `+quoteIdent(fn)+`();`); err != nil { + t.Fatalf("create failure trigger: %v", err) + } + dropTrigger := func() { + f.pool.Exec(context.Background(), `DROP TRIGGER IF EXISTS `+quoteIdent(trigger)+` ON issue`) + f.pool.Exec(context.Background(), `DROP FUNCTION IF EXISTS `+quoteIdent(fn)+`()`) + } + t.Cleanup(dropTrigger) + + // Each attempt: claim, fail, back off. Clear the backoff to reach the next rung. + for attempt := 1; attempt <= len(executorBackoff); attempt++ { + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("attempt %d: %v", attempt, err) + } + s := f.execState(t, execID) + if s.status != "queued" { + t.Fatalf("attempt %d: status = %q (skip=%q err=%q), want queued for retry", + attempt, s.status, s.skipReason, s.errorCode) + } + if !s.retryQueued { + t.Errorf("attempt %d: next_attempt_at was not moved into the future", attempt) + } + if s.actionIndex != 0 { + t.Errorf("attempt %d: action cursor = %d, want 0 (retry resumes at the failed action)", attempt, s.actionIndex) + } + if s.attempts != attempt { + t.Errorf("attempt %d: attempts = %d", attempt, s.attempts) + } + if _, err := f.pool.Exec(ctx, + `UPDATE hook_execution SET next_attempt_at = now() WHERE id = $1`, execID); err != nil { + t.Fatalf("clear backoff: %v", err) + } + } + + // The ladder is spent: the next attempt is terminal. + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("final attempt: %v", err) + } + s := f.execState(t, execID) + if s.status != hookExecFailed { + t.Errorf("status = %q, want failed once the retry ladder is exhausted", s.status) + } + if s.errorCode != errCodeInfra { + t.Errorf("error_code = %q, want %s", s.errorCode, errCodeInfra) + } + + // And the target was never written by any of those attempts. + dropTrigger() + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — a failing action must not partially apply", got) + } +} + +var _ = pgtype.UUID{} + +// --------------------------------------------------------------------------- +// Regressions for the executor review round: independent execution gate, lease +// forward progress, principal-invalid fail-closed, terminal effect audit. +// --------------------------------------------------------------------------- + +// Blocker 1 — execution has its OWN default-off switch, so enabling the engine for +// shadow evaluation cannot start performing real side effects. +func TestExecutionGateIsIndependentOfEventHooksFlag(t *testing.T) { + for _, tc := range []struct { + name string + hooks bool + execution bool + want bool + }{ + {"both off", false, false, false}, + {"hooks on, execution off — shadow mode", true, false, false}, + {"execution on without the engine", false, true, false}, + {"both on", true, true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := featureflag.NewStaticProvider() + provider.Set(featureflags.EventHooks, featureflag.Rule{Default: tc.hooks}) + provider.Set(featureflags.EventHookExecution, featureflag.Rule{Default: tc.execution}) + flags := featureflag.NewService(provider) + if got := featureflags.EventHookExecutionEnabled(context.Background(), flags); got != tc.want { + t.Errorf("EventHookExecutionEnabled = %v, want %v", got, tc.want) + } + }) + } +} + +// Blocker 2 — an execution whose lease keeps elapsing must not hold the head of the +// queue. Elon's repro: the oldest execution reliably outlives its TTL, a healthy one +// sits behind it, and a bounded ClaimAndRun must still reach the healthy one. +func TestExecutorExpiredLeaseDoesNotStarveQueue(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + + // The head execution's action always takes longer than the lease below. + _, stuckExec, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + f.sleepBeforeEffectInsert(t, stuckExec, 0.4) + + // A healthy execution queued behind it: no actions, so it completes instantly. + _, healthyExec, _ := f.seedQueuedExecution(t, `[]`) + if _, err := f.pool.Exec(ctx, + `UPDATE hook_execution SET created_at = now() + interval '1 second' WHERE id = $1`, healthyExec); err != nil { + t.Fatalf("order executions: %v", err) + } + + original := ExecutorLeaseTTL + t.Cleanup(func() { ExecutorLeaseTTL = original }) + ExecutorLeaseTTL = 100 * time.Millisecond + + // Two slots, exactly as the reviewer drove it: without a backoff the stuck row + // consumes both and the healthy one is never reached. + if _, err := f.svc.ClaimAndRun(ctx, 2); err != nil { + t.Fatalf("claim and run: %v", err) + } + + stuck := f.execState(t, stuckExec) + if stuck.status == hookExecSucceeded { + t.Fatalf("the stuck execution unexpectedly succeeded; the lease did not expire") + } + if !stuck.retryQueued { + t.Error("the expired-lease execution was not backed off — it stays the oldest " + + "claimable row and will be re-selected on every tick") + } + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — an expired-lease worker committed its action", got) + } + if s := f.execState(t, healthyExec); s.status != hookExecSucceeded { + t.Errorf("execution behind the stuck one = %q, want succeeded — one execution whose "+ + "lease keeps expiring is starving the whole queue", s.status) + } +} + +// Blocker 3 — if the hook cannot be paused, the skip must NOT be committed. Recording +// the execution as handled while the rule stays enabled is fail-open: it would keep +// producing executions under authority nobody holds. +func TestExecutorPrincipalInvalidFailsClosedWhenPauseFails(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + hookID, execID, _ := f.seedQueuedExecution(t, f.setIssueStatusAction("done")) + + if _, err := f.pool.Exec(ctx, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, f.ws, f.userID); err != nil { + t.Fatalf("remove principal: %v", err) + } + + // Make the pause write fail the way an infrastructure fault would. + trigger := "pause_fail_" + util.UUIDToString(util.NewUUID())[:8] + fn := trigger + "_fn" + if _, err := f.pool.Exec(ctx, ` + CREATE FUNCTION `+quoteIdent(fn)+`() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected pause failure'; END; $$;`); err != nil { + t.Fatalf("create pause-failure function: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + CREATE TRIGGER `+quoteIdent(trigger)+` BEFORE UPDATE OF enabled ON hook + FOR EACH ROW WHEN (NEW.id = `+quoteLiteral(hookID)+`::uuid) + EXECUTE FUNCTION `+quoteIdent(fn)+`();`); err != nil { + t.Fatalf("create pause-failure trigger: %v", err) + } + dropTrigger := func() { + f.pool.Exec(context.Background(), `DROP TRIGGER IF EXISTS `+quoteIdent(trigger)+` ON hook`) + f.pool.Exec(context.Background(), `DROP FUNCTION IF EXISTS `+quoteIdent(fn)+`()`) + } + t.Cleanup(dropTrigger) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("claim and run: %v", err) + } + + // The hook is still enabled because the pause failed — so the execution must NOT + // read as terminally handled. + var enabled bool + if err := f.pool.QueryRow(ctx, `SELECT enabled FROM hook WHERE id = $1`, hookID).Scan(&enabled); err != nil { + t.Fatal(err) + } + if !enabled { + t.Fatal("the pause unexpectedly succeeded; the failure injection did not take") + } + if s := f.execState(t, execID); s.status == hookExecSkipped { + t.Error("execution was finalized as skipped while its hook stayed enabled — " + + "fail-open: the rule keeps producing executions under a departed principal") + } + + // Once the pause can be written, the outcome converges: hook paused, execution + // terminally skipped, and the admins who can re-arm it are notified. + dropTrigger() + if _, err := f.pool.Exec(ctx, + `UPDATE hook_execution SET next_attempt_at = now() WHERE id = $1`, execID); err != nil { + t.Fatalf("clear backoff: %v", err) + } + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("second run: %v", err) + } + + var reason string + if err := f.pool.QueryRow(ctx, + `SELECT enabled, COALESCE(disabled_reason,'') FROM hook WHERE id = $1`, hookID).Scan(&enabled, &reason); err != nil { + t.Fatal(err) + } + if enabled || reason != hookDisabledPrincipalInvalid { + t.Errorf("hook = (enabled=%v, reason=%q), want paused with %s", enabled, reason, hookDisabledPrincipalInvalid) + } + if s := f.execState(t, execID); s.status != hookExecSkipped || s.skipReason != skipPrincipalInvalid { + t.Errorf("execution = (%q, %q), want skipped/%s", s.status, s.skipReason, skipPrincipalInvalid) + } + if got := f.issueStatus(t); got != "todo" { + t.Errorf("issue status = %q, want todo — the action ran under a departed principal", got) + } +} + +// Blocker 4 — a terminal effect survives a restart as exactly ONE row per +// (execution, action index): re-running must update the audit row, never duplicate it. +func TestExecutorTerminalEffectIsSingleRowAcrossRetries(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + foreign := uuid.NewString() + actions := `[{"type":"set_issue_status","issue_id":"` + foreign + `","status":"done"}]` + _, execID, _ := f.seedQueuedExecution(t, actions) + + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("first run: %v", err) + } + if effects := f.effectRows(t, execID); len(effects) != 1 || effects[0] != hookExecSkipped { + t.Fatalf("effects after first run = %v, want one skipped row", effects) + } + + // Replay the same execution, as a restart that lost the terminal write would. + if _, err := f.pool.Exec(ctx, ` + UPDATE hook_execution + SET status = 'queued', skip_reason = NULL, completed_at = NULL, + lease_token = NULL, lease_expires_at = NULL, next_attempt_at = now() + WHERE id = $1`, execID); err != nil { + t.Fatalf("replay execution: %v", err) + } + if _, err := f.svc.ClaimAndRun(ctx, 5); err != nil { + t.Fatalf("second run: %v", err) + } + + effects := f.effectRows(t, execID) + if len(effects) != 1 || effects[0] != hookExecSkipped { + t.Errorf("effects after replay = %v, want still exactly one skipped row for "+ + "(execution, action 0)", effects) + } + var attempts int + if err := f.pool.QueryRow(ctx, + `SELECT attempts FROM hook_action_effect WHERE execution_id = $1`, execID).Scan(&attempts); err != nil { + t.Fatal(err) + } + if attempts < 2 { + t.Errorf("effect attempts = %d, want the replay counted on the same row", attempts) + } +} + +// sleepBeforeEffectInsert makes one execution's action effect insert pause, so the +// action reliably outlives a short lease TTL. +func (f matcherFixture) sleepBeforeEffectInsert(t *testing.T, execID string, seconds float64) { + t.Helper() + ctx := context.Background() + name := fmt.Sprintf("hook_effect_sleep_%d", time.Now().UnixNano()) + fn := name + "_fn" + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN PERFORM pg_sleep(%f); RETURN NEW; END; $$;`, quoteIdent(fn), seconds)); err != nil { + t.Fatalf("create effect sleep function: %v", err) + } + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE TRIGGER %s BEFORE INSERT ON hook_action_effect + FOR EACH ROW WHEN (NEW.execution_id = %s::uuid) + EXECUTE FUNCTION %s();`, quoteIdent(name), quoteLiteral(execID), quoteIdent(fn))); err != nil { + t.Fatalf("create effect sleep trigger: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, fmt.Sprintf("DROP TRIGGER IF EXISTS %s ON hook_action_effect", quoteIdent(name))) + f.pool.Exec(bg, fmt.Sprintf("DROP FUNCTION IF EXISTS %s()", quoteIdent(fn))) + }) +} diff --git a/server/internal/service/hook_matcher.go b/server/internal/service/hook_matcher.go new file mode 100644 index 00000000000..e5128b0bb27 --- /dev/null +++ b/server/internal/service/hook_matcher.go @@ -0,0 +1,554 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The durable matcher (MUL-4332 PR3). It consumes pending domain_event rows and, +// for each enabled hook whose active revision listens to that event, runs the SAME +// automation.Evaluate as dry-run/explain, then completes the fire decision the +// read-only evaluator deliberately left open: the depth guard, the rising-edge +// latch, and per_event/rising_edge fire/skip. Each decision is persisted as one +// hook_execution row carrying the evaluator's structured snapshots and the pinned +// revision, idempotent per (hook, event). +// +// CLAIM AND PIN ARE THE SAME INSTANT. One transaction claims the event, materializes +// the complete (hook, revision) candidate set, decides every candidate, and +// finalizes the event. Two properties make the pin real rather than nominal: +// +// - The claim runs INSIDE this transaction, so there is no window between leasing +// the event and choosing its revisions in which an edit could land (§5.1: +// "使用 matcher claim 时的当前 enabled revision"). +// - The candidate set comes from ONE statement. Under READ COMMITTED each +// statement gets a fresh snapshot, so reading candidate ids and then re-reading +// each hook's active_revision_id would let two candidates in the same +// transaction be decided against revisions from different instants. The matcher +// never re-reads active_revision_id after the pin. +// +// Ownership is asserted with one predicate — right token, still 'dispatching', and +// not expired under DATABASE clock time — used identically at entry and at the +// finalize CAS. A worker whose lease expired mid-decision therefore commits nothing. +// +// Inside the transaction each candidate runs in its own SAVEPOINT, so a revision +// whose stored config cannot be parsed is isolated as one terminal `failed` row +// instead of starving the healthy rules on the same event. +// +// This slice does NOT run actions — a fired hook lands `queued` for the executor +// (a later slice). With no executor and the automation_event_hooks flag off +// (the matcher loop is gated on it), production behaviour is unchanged. + +const ( + hookExecQueued = "queued" + hookExecSkipped = "skipped" + hookExecFailed = "failed" + + // Stable skip_reason vocabulary, matching the ratified contract (§5.2 / §15.3). + skipMaxDepth = "max_depth" + skipConditionFalse = "condition_false" + skipConditionAlreadyTrue = "condition_already_true" + + // errCodeInvalidConfig marks the isolation row written for a candidate whose + // stored revision cannot be evaluated. + errCodeInvalidConfig = "invalid_config" + + // latchStateKind keys rising-edge latches in automation_state; one row per hook. + latchStateKind = "hook_edge" + + // maxHopCount is the loop-depth guard (§15.3: "hop_count 上限 8；超过上限的候选记 + // skipped(max_depth)"). The bound is INCLUSIVE: an event at hop_count == 8 is AT + // the limit and still fires; only hop_count > 8 exceeds it and is skipped. + maxHopCount = 8 + + // MatcherBatchSize bounds how many events one matcher tick claims and decides. + MatcherBatchSize = 100 + + // matcherFailureBackoff delays an event that failed transiently, so it cannot sit + // at the head of the queue re-failing and starving everything behind it. + matcherFailureBackoff = 30 * time.Second +) + +// MatcherLeaseTTL bounds how long one event's decision may take. It is a var so +// tests can drive the expired-lease path deterministically. +var MatcherLeaseTTL = 2 * time.Minute + +// errLeaseLost means this worker is not (or is no longer) the event's owner — +// wrong token, already finalized, or an expired lease. It always aborts the +// transaction, so a non-owner commits nothing. It is an expected outcome, not a +// failure, and never escapes ClaimAndMatch as an error. +var errLeaseLost = errors.New("hook matcher: event lease lost") + +// errNoClaimableEvent ends a tick: the queue has nothing available right now. +var errNoClaimableEvent = errors.New("hook matcher: no claimable event") + +// pinnedCandidate is one (hook, revision) pair fixed at claim time, together with +// the revision configuration the evaluator needs. The matcher decides against this +// and never re-reads hook.active_revision_id, which is what keeps a revision edit +// committed after the claim from changing this event's decision. +type pinnedCandidate struct { + HookID pgtype.UUID + RevisionID pgtype.UUID + Match []byte + Conditions []byte + FireMode string +} + +// latchState is the persisted rising-edge latch. RevisionID pins it to a +// revision so a config change starts a fresh edge rather than inheriting a stale +// satisfied flag. +type latchState struct { + Satisfied bool `json:"satisfied"` + RevisionID string `json:"revision_id"` +} + +// hookConfigError wraps a DETERMINISTIC per-candidate config failure. It carries the +// revision it was pinned to so the caller can record the isolation row after rolling +// that candidate's savepoint back. Transient (database) failures are never wrapped +// in it, so the two can never be confused. +type hookConfigError struct { + revisionID pgtype.UUID + err error +} + +func (e *hookConfigError) Error() string { return e.err.Error() } +func (e *hookConfigError) Unwrap() error { return e.err } + +// ClaimAndMatch claims and decides up to batchSize events, one authoritative +// transaction each, and returns how many were finalized as dispatched. +func (s *HookService) ClaimAndMatch(ctx context.Context, batchSize int32) (int, error) { + dispatched := 0 + for i := int32(0); i < batchSize; i++ { + claimed, ok, err := s.claimAndDecideOne(ctx) + if err != nil { + // The event rolled back to pending and has been backed off; stop this + // tick rather than immediately re-claiming the same head-of-queue row. + slog.Warn("hook matcher: event failed, deferred for retry", "error", err) + return dispatched, nil + } + if !claimed { + break // queue drained + } + if ok { + dispatched++ + } + } + return dispatched, nil +} + +// claimAndDecideOne is the matcher's authoritative unit of work: one transaction +// that claims an event, pins its candidate revisions, writes every decision and +// latch, and finalizes the event. It reports whether an event was claimed at all +// and whether it was finalized as dispatched. +func (s *HookService) claimAndDecideOne(ctx context.Context) (claimed bool, dispatched bool, err error) { + lease := util.NewUUID() + var eventID pgtype.UUID + err = s.inTxWith(ctx, func(tx pgx.Tx, qtx *db.Queries) error { + rows, err := qtx.ClaimOneEventWithCandidates(ctx, db.ClaimOneEventWithCandidatesParams{ + LeaseToken: lease, + LeaseTtlSeconds: MatcherLeaseTTL.Seconds(), + }) + if err != nil { + return err + } + if len(rows) == 0 { + return errNoClaimableEvent + } + event := claimedEvent(rows[0]) + claimed, eventID = true, event.ID + ok, err := s.decideAndFinalize(ctx, tx, qtx, event, pinnedCandidates(rows), lease) + dispatched = ok + return err + }) + switch { + case errors.Is(err, errNoClaimableEvent): + return false, false, nil + case errors.Is(err, errLeaseLost): + if !claimed { + // We never took the event, so it is not ours to back off. + return false, false, nil + } + // We DID claim it in this transaction and then lost ownership, which — with + // the claim and the decision in one transaction and the event row locked + // throughout — means our own fresh lease expired mid-decision. The rollback + // also undid the claim, restoring the original available_at, so without a + // backoff the next tick selects this same oldest event again and everything + // behind it starves. Back it off exactly like any other failed decision. + if derr := s.deferFailedEvent(ctx, eventID); derr != nil { + slog.Warn("hook matcher: could not back off an expired-lease event", + "event_id", util.UUIDToString(eventID), "error", derr) + return false, false, nil // end the tick rather than spin on the same row + } + slog.Warn("hook matcher: lease expired mid-decision, event backed off", + "event_id", util.UUIDToString(eventID)) + return true, false, nil // deferred — keep draining the rest of the queue + case err != nil: + if derr := s.deferFailedEvent(ctx, eventID); derr != nil { + slog.Warn("hook matcher: could not back off a failed event", + "event_id", util.UUIDToString(eventID), "error", derr) + } + return claimed, false, err + } + return claimed, dispatched, nil +} + +// claimedEvent rebuilds the event envelope the decision needs from a claim row. +func claimedEvent(r db.ClaimOneEventWithCandidatesRow) db.DomainEvent { + return db.DomainEvent{ + ID: r.EventID, + WorkspaceID: r.EventWorkspaceID, + Type: r.EventType, + SubjectID: r.EventSubjectID, + ActorType: r.EventActorType, + ActorID: r.EventActorID, + Payload: r.EventPayload, + CorrelationID: r.EventCorrelationID, + HopCount: r.EventHopCount, + } +} + +// pinnedCandidates extracts the candidate set from a claim result. An event with no +// candidates comes back as a single row with no hook, which yields an empty set. +func pinnedCandidates(rows []db.ClaimOneEventWithCandidatesRow) []pinnedCandidate { + out := make([]pinnedCandidate, 0, len(rows)) + for _, r := range rows { + if !r.HookID.Valid { + continue + } + out = append(out, pinnedCandidate{ + HookID: r.HookID, + RevisionID: r.RevisionID, + Match: r.Match, + Conditions: r.Conditions, + FireMode: r.FireMode, + }) + } + return out +} + +// decideAndFinalize asserts ownership, decides every pinned candidate, and +// finalizes — all within the caller's claim transaction. +func (s *HookService) decideAndFinalize(ctx context.Context, tx pgx.Tx, qtx *db.Queries, event db.DomainEvent, candidates []pinnedCandidate, lease pgtype.UUID) (bool, error) { + // Ownership, fail-closed, BEFORE any write. Same predicate as the finalize CAS, + // including the not-expired condition under database clock time. + owned, err := qtx.GetOwnedDomainEventForDispatch(ctx, db.GetOwnedDomainEventForDispatchParams{ + ID: event.ID, LeaseToken: lease, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, errLeaseLost + } + return false, err + } + + // Project the event once, so every candidate sees the same view. A payload the + // matcher can never decode fails identically on every retry, so it is terminal + // rather than re-leased forever. + view, err := eventToView(owned) + if err != nil { + rows, ferr := qtx.MarkDomainEventFailed(ctx, db.MarkDomainEventFailedParams{ID: owned.ID, LeaseToken: lease}) + if ferr != nil { + return false, ferr + } + if rows != 1 { + return false, errLeaseLost + } + slog.Warn("hook matcher: undecodable event payload, marked failed", "event_id", util.UUIDToString(owned.ID), "error", err) + return false, nil + } + + if err := s.decideCandidates(ctx, tx, qtx, owned, view, candidates); err != nil { + return false, err + } + + rows, err := qtx.MarkDomainEventDispatched(ctx, db.MarkDomainEventDispatchedParams{ID: owned.ID, LeaseToken: lease}) + if err != nil { + return false, err + } + if rows != 1 { + return false, errLeaseLost + } + return true, nil +} + +// MatchEvent decides every candidate hook for one event, without claiming or +// finalizing it. The production path is ClaimAndMatch, which pins the candidate set +// in the SAME statement that claims the event; this entry point resolves candidates +// separately and exists for direct decision tests. +func (s *HookService) MatchEvent(ctx context.Context, event db.DomainEvent) error { + view, err := eventToView(event) + if err != nil { + return err + } + return s.inTxWith(ctx, func(tx pgx.Tx, qtx *db.Queries) error { + rows, err := qtx.ListActiveHookRevisionsForEvent(ctx, db.ListActiveHookRevisionsForEventParams{ + WorkspaceID: event.WorkspaceID, + EventType: event.Type, + }) + if err != nil { + return err + } + candidates := make([]pinnedCandidate, 0, len(rows)) + for _, r := range rows { + candidates = append(candidates, pinnedCandidate{ + HookID: r.HookID, + RevisionID: r.RevisionID, + Match: r.Match, + Conditions: r.Conditions, + FireMode: r.FireMode, + }) + } + return s.decideCandidates(ctx, tx, qtx, event, view, candidates) + }) +} + +// decideCandidates decides each already-pinned candidate within the caller's +// transaction. Each candidate runs in its own SAVEPOINT so one unusable revision can +// neither abort the transaction nor block the rest: a deterministic config failure is +// recorded as a terminal `failed` row and the loop continues, while a transient +// (database) failure aborts the whole event so it retries intact. +func (s *HookService) decideCandidates(ctx context.Context, tx pgx.Tx, qtx *db.Queries, event db.DomainEvent, view automation.EventView, candidates []pinnedCandidate) error { + for _, candidate := range candidates { + sp, err := tx.Begin(ctx) // SAVEPOINT + if err != nil { + return err + } + err = processHookForEvent(ctx, s.Queries.WithTx(sp), event, view, candidate) + if err == nil { + if err := sp.Commit(ctx); err != nil { // RELEASE SAVEPOINT + return err + } + continue + } + // Undo only this candidate's partial work; the event's transaction lives on. + if rberr := sp.Rollback(ctx); rberr != nil { + return rberr + } + var cfgErr *hookConfigError + if !errors.As(err, &cfgErr) { + return err // transient — retry the whole event rather than lose a decision + } + if err := writeHookExecutionFailure(ctx, qtx, event, candidate.HookID, cfgErr); err != nil { + return err + } + slog.Warn("hook matcher: candidate isolated, unusable revision", + "event_id", util.UUIDToString(event.ID), "hook_id", util.UUIDToString(candidate.HookID), "error", cfgErr) + } + return nil +} + +// processHookForEvent makes and persists the fire/skip decision for one (hook, +// event) pair against the revision pinned when the event was claimed. +func processHookForEvent(ctx context.Context, qtx *db.Queries, event db.DomainEvent, view automation.EventView, candidate pinnedCandidate) error { + // Serialize this hook's latch read-modify-write against other matchers. The + // revision is NOT re-read here — the pinned one below is authoritative. + if _, err := qtx.LockHookForDecision(ctx, db.LockHookForDecisionParams{ + ID: candidate.HookID, WorkspaceID: event.WorkspaceID, + }); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil // hook vanished since the pin + } + return err + } + + rev, err := pinnedRevisionToEval(candidate, event.Type) + if err != nil { + return &hookConfigError{ + revisionID: candidate.RevisionID, + err: fmt.Errorf("%w: parse stored revision: %v", automation.ErrInvalidConfig, err), + } + } + ev, err := automation.Evaluate(ctx, view, rev, &issueStateReader{q: qtx, workspaceID: event.WorkspaceID}) + if err != nil { + if errors.Is(err, automation.ErrInvalidConfig) { + return &hookConfigError{revisionID: candidate.RevisionID, err: err} + } + return err // transient state-read failure + } + + // A non-matching hook is not a candidate — write nothing. + if !ev.Matched { + return nil + } + + matchSnap, err := ev.MatchSnapshot() + if err != nil { + return err + } + condSnap, err := ev.ConditionSnapshot() + if err != nil { + return err + } + + // The depth guard decides only whether THIS event may fire. It never suppresses + // the condition state a matched event observed (review point 3). + overDepth := event.HopCount > maxHopCount + + if candidate.FireMode != automation.FireRisingEdge { + // per_event: fire whenever the conditions currently hold. + status, reason := hookExecQueued, "" + switch { + case overDepth: + status, reason = hookExecSkipped, skipMaxDepth + case !ev.ConditionsMet: + status, reason = hookExecSkipped, skipConditionFalse + } + _, err := writeHookExecution(ctx, qtx, event, candidate, status, reason, matchSnap, condSnap) + return err + } + + // rising_edge: fire only on a false→true transition of the latch. + prev, err := readLatch(ctx, qtx, event.WorkspaceID, candidate.HookID, candidate.RevisionID) + if err != nil { + return err + } + nowSatisfied := ev.ConditionsMet + + status, reason := hookExecQueued, "" + switch { + case overDepth: + status, reason = hookExecSkipped, skipMaxDepth + case !nowSatisfied: + status, reason = hookExecSkipped, skipConditionFalse + case prev: + status, reason = hookExecSkipped, skipConditionAlreadyTrue + } + + inserted, err := writeHookExecution(ctx, qtx, event, candidate, status, reason, matchSnap, condSnap) + if err != nil { + return err + } + // Record the condition state this matched event observed, exactly once per + // (hook, event) — INCLUDING when the depth guard rejected the fire. Skipping the + // advance there would strand the latch and swallow the next legitimate false→true + // edge. Gating on `inserted` keeps a re-leased retry from double-advancing it. + if inserted { + return upsertLatch(ctx, qtx, event.WorkspaceID, candidate.HookID, candidate.RevisionID, nowSatisfied) + } + return nil +} + +// pinnedRevisionToEval builds the evaluator's view of the revision pinned at claim +// time. eventType is the type the candidate query already matched on. +func pinnedRevisionToEval(candidate pinnedCandidate, eventType string) (automation.EvalRevision, error) { + var conds []automation.ConditionSpec + if len(candidate.Conditions) > 0 { + if err := json.Unmarshal(candidate.Conditions, &conds); err != nil { + return automation.EvalRevision{}, err + } + } + return automation.EvalRevision{ + EventType: eventType, + Match: candidate.Match, + Conditions: conds, + FireMode: candidate.FireMode, + }, nil +} + +// deferFailedEvent backs an event off after a failed decision so it cannot hold the +// head of the queue. It runs on its own connection, outside the rolled-back decision +// transaction, and detached from ctx so a cancelled tick still records the backoff. +func (s *HookService) deferFailedEvent(ctx context.Context, eventID pgtype.UUID) error { + if !eventID.Valid { + return nil + } + _, err := s.Queries.DeferDomainEventDispatch(context.WithoutCancel(ctx), db.DeferDomainEventDispatchParams{ + ID: eventID, + BackoffSeconds: int32(matcherFailureBackoff.Seconds()), + }) + return err +} + +// writeHookExecution inserts one decision row idempotently, pinned to the revision +// chosen at claim time. It reports whether a new row was created (false means it +// already existed — a re-processed event). +func writeHookExecution(ctx context.Context, qtx *db.Queries, event db.DomainEvent, candidate pinnedCandidate, status, skipReason string, matchSnap, condSnap []byte) (bool, error) { + reason := pgtype.Text{} + if skipReason != "" { + reason = pgtype.Text{String: skipReason, Valid: true} + } + _, err := qtx.CreateHookExecution(ctx, db.CreateHookExecutionParams{ + ID: util.NewUUID(), + WorkspaceID: event.WorkspaceID, + HookID: candidate.HookID, + HookRevisionID: candidate.RevisionID, + EventID: event.ID, + CorrelationID: event.CorrelationID, + Status: status, + SkipReason: reason, + MatchSnapshot: matchSnap, + ConditionSnapshot: condSnap, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil // ON CONFLICT DO NOTHING → already processed + } + return false, err + } + return true, nil +} + +// writeHookExecutionFailure records the terminal isolation row for a candidate whose +// stored revision could not be evaluated, so the event can finalize and the healthy +// candidates keep their decisions. +func writeHookExecutionFailure(ctx context.Context, qtx *db.Queries, event db.DomainEvent, hookID pgtype.UUID, cfgErr *hookConfigError) error { + _, err := qtx.CreateHookExecutionFailure(ctx, db.CreateHookExecutionFailureParams{ + ID: util.NewUUID(), + WorkspaceID: event.WorkspaceID, + HookID: hookID, + HookRevisionID: cfgErr.revisionID, + EventID: event.ID, + CorrelationID: event.CorrelationID, + ErrorCode: pgtype.Text{String: errCodeInvalidConfig, Valid: true}, + Error: pgtype.Text{String: cfgErr.Error(), Valid: true}, + }) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err // ErrNoRows → the isolation row already exists + } + return nil +} + +// readLatch returns the previous satisfied state of a rising-edge latch, treating +// a latch pinned to a different revision as fresh (not satisfied). +func readLatch(ctx context.Context, qtx *db.Queries, workspaceID, hookID, revisionID pgtype.UUID) (bool, error) { + row, err := qtx.GetAutomationStateForUpdate(ctx, db.GetAutomationStateForUpdateParams{ + WorkspaceID: workspaceID, StateKind: latchStateKind, StateKey: util.UUIDToString(hookID), + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, err + } + var st latchState + if len(row.State) > 0 { + if err := json.Unmarshal(row.State, &st); err != nil { + return false, err + } + } + if st.RevisionID != util.UUIDToString(revisionID) { + return false, nil // stale latch from a superseded revision + } + return st.Satisfied, nil +} + +func upsertLatch(ctx context.Context, qtx *db.Queries, workspaceID, hookID, revisionID pgtype.UUID, satisfied bool) error { + state, err := json.Marshal(latchState{Satisfied: satisfied, RevisionID: util.UUIDToString(revisionID)}) + if err != nil { + return err + } + _, err = qtx.UpsertAutomationState(ctx, db.UpsertAutomationStateParams{ + WorkspaceID: workspaceID, StateKind: latchStateKind, StateKey: util.UUIDToString(hookID), State: state, + }) + return err +} diff --git a/server/internal/service/hook_matcher_test.go b/server/internal/service/hook_matcher_test.go new file mode 100644 index 00000000000..1b473e69d5f --- /dev/null +++ b/server/internal/service/hook_matcher_test.go @@ -0,0 +1,1080 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +type matcherFixture struct { + svc *HookService + pool *pgxpool.Pool + ws string + userID string + issueID string +} + +func newMatcherFixture(t *testing.T) matcherFixture { + t.Helper() + pool := newTaskClaimRacePool(t) // skips if no DB + ws, userID, _, issueID := seedAttributionFixture(t, pool) + return matcherFixture{ + svc: NewHookService(db.New(pool), pool), + pool: pool, + ws: ws, + userID: userID, + issueID: issueID, + } +} + +// seedHook inserts a hook + immutable revision #1 directly (the matcher only +// reads them). Returns the hook id. +func (f matcherFixture) seedHook(t *testing.T, eventType, matchJSON, condJSON, fireMode string) string { + t.Helper() + ctx := context.Background() + hookID, revID := uuid.NewString(), uuid.NewString() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook (id, workspace_id, name, enabled, active_revision_id, scope_type, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id) + VALUES ($1, $2, 'm hook', true, $3, 'workspace', 'user', 'member', $4, $4)`, + hookID, f.ws, revID, f.userID); err != nil { + t.Fatalf("seed hook: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook_revision (id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id) + VALUES ($1, $2, 1, $3, $4::jsonb, $5::jsonb, $6, '[]'::jsonb, 'member', $7)`, + revID, hookID, eventType, matchJSON, condJSON, fireMode, f.userID); err != nil { + t.Fatalf("seed hook_revision: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, `DELETE FROM hook_execution WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM automation_state WHERE workspace_id = $1 AND state_key = $2`, f.ws, hookID) + f.pool.Exec(bg, `DELETE FROM hook_revision WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM hook WHERE id = $1`, hookID) + }) + return hookID +} + +// seedEvent inserts a pending issue.status_changed domain event and returns it. +func (f matcherFixture) seedEvent(t *testing.T, to string, hopCount int) db.DomainEvent { + t.Helper() + var id string + payload := `{"from":"in_progress","to":"` + to + `"}` + if err := f.pool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, hop_count) + VALUES ($1, 'issue.status_changed', 1, 'issue', $2, 'member', $3, $4::jsonb, gen_random_uuid(), $5) + RETURNING id`, f.ws, f.issueID, f.userID, payload, hopCount).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { f.pool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + ev, err := f.svc.Queries.GetDomainEvent(context.Background(), util.MustParseUUID(id)) + if err != nil { + t.Fatalf("load event: %v", err) + } + return ev +} + +func (f matcherFixture) setIssueStatus(t *testing.T, status string) { + t.Helper() + if _, err := f.pool.Exec(context.Background(), `UPDATE issue SET status = $1 WHERE id = $2`, status, f.issueID); err != nil { + t.Fatalf("set issue status: %v", err) + } +} + +// seedHookAged is seedHook with an explicit age, so candidate order +// (the candidate query sorts by created_at ASC) is deterministic. An older +// hook is evaluated first. +func (f matcherFixture) seedHookAged(t *testing.T, eventType, matchJSON, condJSON, fireMode, age string) string { + t.Helper() + hookID := f.seedHook(t, eventType, matchJSON, condJSON, fireMode) + if _, err := f.pool.Exec(context.Background(), + `UPDATE hook SET created_at = now() - $2::interval WHERE id = $1`, hookID, age); err != nil { + t.Fatalf("age hook: %v", err) + } + return hookID +} + +// claimWithLease puts an event into the state the matcher's claim leaves it in and +// returns the lease that owns it, so a test can drive the decision as either the real +// owner or a stale holder. +func (f matcherFixture) claimWithLease(t *testing.T, eventID pgtype.UUID) pgtype.UUID { + t.Helper() + lease := util.NewUUID() + if _, err := f.pool.Exec(context.Background(), ` + UPDATE domain_event + SET dispatch_status = 'dispatching', lease_token = $2, + lease_expires_at = now() + interval '5 minutes', attempts = attempts + 1 + WHERE id = $1`, util.UUIDToString(eventID), util.UUIDToString(lease)); err != nil { + t.Fatalf("claim event: %v", err) + } + return lease +} + +// decideClaimed drives ownership + decision + finalize for one already-claimed +// event. Production claims, pins and decides in a single transaction +// (claimAndDecideOne); this resolves candidates separately so a test can target a +// specific event, but runs the very same decideAndFinalize. +func (f matcherFixture) decideClaimed(ctx context.Context, event db.DomainEvent, lease pgtype.UUID) (bool, error) { + var dispatched bool + var err error + err = f.svc.inTxWith(ctx, func(tx pgx.Tx, qtx *db.Queries) error { + rows, err := qtx.ListActiveHookRevisionsForEvent(ctx, db.ListActiveHookRevisionsForEventParams{ + WorkspaceID: event.WorkspaceID, EventType: event.Type, + }) + if err != nil { + return err + } + candidates := make([]pinnedCandidate, 0, len(rows)) + for _, r := range rows { + candidates = append(candidates, pinnedCandidate{ + HookID: r.HookID, RevisionID: r.RevisionID, + Match: r.Match, Conditions: r.Conditions, FireMode: r.FireMode, + }) + } + ok, err := f.svc.decideAndFinalize(ctx, tx, qtx, event, candidates, lease) + dispatched = ok + return err + }) + if errors.Is(err, errLeaseLost) { + return false, nil + } + return dispatched, err +} + +type eventState struct { + status string + lease string // "" when NULL + hasDispatchedAt bool +} + +func (f matcherFixture) eventState(t *testing.T, eventID pgtype.UUID) eventState { + t.Helper() + var s eventState + if err := f.pool.QueryRow(context.Background(), ` + SELECT dispatch_status, COALESCE(lease_token::text, ''), dispatched_at IS NOT NULL + FROM domain_event WHERE id = $1`, util.UUIDToString(eventID)). + Scan(&s.status, &s.lease, &s.hasDispatchedAt); err != nil { + t.Fatalf("load event state: %v", err) + } + return s +} + +// latchFor reads the persisted rising-edge latch for a hook. +func (f matcherFixture) latchFor(t *testing.T, hookID string) (satisfied, found bool) { + t.Helper() + err := f.pool.QueryRow(context.Background(), + `SELECT (state->>'satisfied')::bool FROM automation_state + WHERE workspace_id = $1 AND state_kind = $2 AND state_key = $3`, + f.ws, latchStateKind, hookID).Scan(&satisfied) + if err != nil { + return false, false + } + return satisfied, true +} + +type execRow struct { + status string + skipReason string + errorCode string + revisionID string + matchSnap []byte + condSnap []byte + count int +} + +func (f matcherFixture) execFor(t *testing.T, hookID string) execRow { + t.Helper() + var r execRow + if err := f.pool.QueryRow(context.Background(), + `SELECT count(*) FROM hook_execution WHERE hook_id = $1`, hookID).Scan(&r.count); err != nil { + t.Fatalf("count executions: %v", err) + } + if r.count == 0 { + return r + } + if err := f.pool.QueryRow(context.Background(), ` + SELECT status, COALESCE(skip_reason,''), COALESCE(error_code,''), hook_revision_id::text, match_snapshot, condition_snapshot + FROM hook_execution WHERE hook_id = $1 ORDER BY created_at DESC LIMIT 1`, hookID). + Scan(&r.status, &r.skipReason, &r.errorCode, &r.revisionID, &r.matchSnap, &r.condSnap); err != nil { + t.Fatalf("load execution: %v", err) + } + return r +} + +func TestMatcherPerEventFires(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + r := f.execFor(t, hookID) + if r.count != 1 || r.status != hookExecQueued { + t.Fatalf("expected 1 queued execution, got count=%d status=%q", r.count, r.status) + } + if len(r.matchSnap) == 0 || len(r.condSnap) == 0 { + t.Errorf("execution is missing snapshots") + } +} + +func TestMatcherConditionFalseSkips(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FirePerEvent) + event := f.seedEvent(t, "in_progress", 0) + + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.status != hookExecSkipped || r.skipReason != skipConditionFalse { + t.Fatalf("expected skipped condition_false, got %+v", r) + } +} + +// The depth bound is INCLUSIVE (§15.3: "上限 8；超过上限的候选记 skipped(max_depth)"). +// hop_count == 8 is AT the limit and still fires; only hop_count > 8 exceeds it. +// Pinning both sides makes the boundary explicit rather than an artifact of >= vs >. +func TestMatcherHopGuardBoundary(t *testing.T) { + for _, tc := range []struct { + name string + hop int + wantStatus string + wantReason string + }{ + {"at the limit fires", maxHopCount, hookExecQueued, ""}, + {"over the limit is skipped", maxHopCount + 1, hookExecSkipped, skipMaxDepth}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", tc.hop)); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.status != tc.wantStatus || r.skipReason != tc.wantReason { + t.Fatalf("hop_count=%d: want status=%q reason=%q, got %+v", tc.hop, tc.wantStatus, tc.wantReason, r) + } + }) + } +} + +// The rising-edge latch: fires on false→true, holds while true, resets on false, +// then fires again on the next false→true. +func TestMatcherRisingEdgeLatch(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + + assertLast := func(want, reason string) { + t.Helper() + if r := f.execFor(t, hookID); r.status != want || r.skipReason != reason { + t.Fatalf("want status=%q reason=%q, got %+v", want, reason, r) + } + } + + // false→true: condition satisfied for the first time → fires. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecQueued, "") + + // still true: another matching event → no new edge. + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecSkipped, skipConditionAlreadyTrue) + + // condition drops to false → latch resets. + f.setIssueStatus(t, "todo") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "todo", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecSkipped, skipConditionFalse) + + // false→true again → fires again. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecQueued, "") +} + +// Re-processing the same event must not double-create the execution or double- +// advance the latch. +func TestMatcherIdempotent(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + f.setIssueStatus(t, "done") + event := f.seedEvent(t, "done", 0) + + for i := 0; i < 3; i++ { + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match %d: %v", i, err) + } + } + if r := f.execFor(t, hookID); r.count != 1 || r.status != hookExecQueued { + t.Fatalf("re-processing created %d rows (want 1), status=%q", r.count, r.status) + } + // The latch advanced exactly once: a further, distinct event must NOT re-fire. + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + if r := f.execFor(t, hookID); r.status != hookExecSkipped || r.skipReason != skipConditionAlreadyTrue { + t.Fatalf("latch double-advanced: next event fired again (%+v)", r) + } +} + +// Hard acceptance: the matcher's stored snapshots are byte-identical to the shared +// evaluator's output for the same (event, revision, state) — the same result +// explain returns, so an explanation can never drift from a real decision. +func TestMatcherSnapshotConsistency(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, cond, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + // Compute the expected snapshots through the SAME evaluator path explain uses. + hook, err := f.svc.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: util.MustParseUUID(hookID), WorkspaceID: util.MustParseUUID(f.ws)}) + if err != nil { + t.Fatal(err) + } + rawRev, err := f.svc.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + t.Fatal(err) + } + rev, _ := revisionToEval(rawRev) + view, _ := eventToView(event) + ev, err := automation.Evaluate(ctx, view, rev, &issueStateReader{q: f.svc.Queries, workspaceID: util.MustParseUUID(f.ws)}) + if err != nil { + t.Fatal(err) + } + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + r := f.execFor(t, hookID) + + // The snapshot column is jsonb, so Postgres re-serializes it (key order / + // whitespace); compare the PARSED structured content against the evaluator's + // output — that is the drift-free contract that matters. + var gotMatch []automation.ClauseResult + if err := json.Unmarshal(r.matchSnap, &gotMatch); err != nil { + t.Fatalf("parse stored match_snapshot: %v", err) + } + if !reflect.DeepEqual(gotMatch, ev.MatchClauses) { + t.Errorf("match_snapshot content differs from evaluator:\n stored=%+v\n eval =%+v", gotMatch, ev.MatchClauses) + } + var gotCond []automation.ConditionResult + if err := json.Unmarshal(r.condSnap, &gotCond); err != nil { + t.Fatalf("parse stored condition_snapshot: %v", err) + } + if !reflect.DeepEqual(gotCond, ev.Conditions) { + t.Errorf("condition_snapshot content differs from evaluator:\n stored=%+v\n eval =%+v", gotCond, ev.Conditions) + } + if r.revisionID != util.UUIDToString(hook.ActiveRevisionID) { + t.Errorf("revision not pinned: stored=%s active=%s", r.revisionID, util.UUIDToString(hook.ActiveRevisionID)) + } +} + +// ClaimAndMatch leases a pending event, matches it, and marks it dispatched. +func TestMatcherClaimAndMatchDispatches(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + // ClaimAndMatch is a GLOBAL outbox consumer (not workspace-scoped) and the + // shared CI test DB carries a backlog of undispatched events from other tests. + // Our event has the newest (highest) seq, so drain in bounded batches until it + // is reached — its seq is fixed, and concurrently-added events have higher seq, + // so this converges. + var status string + for i := 0; i < 60; i++ { + if _, err := f.svc.ClaimAndMatch(ctx, 500); err != nil { + t.Fatalf("claim and match: %v", err) + } + f.pool.QueryRow(ctx, `SELECT dispatch_status FROM domain_event WHERE id = $1`, util.UUIDToString(event.ID)).Scan(&status) + if status == "dispatched" { + break + } + } + if status != "dispatched" { + t.Fatalf("event dispatch_status = %q after draining, want dispatched", status) + } + if r := f.execFor(t, hookID); r.status != hookExecQueued { + t.Errorf("expected queued execution after claim+match, got %+v", r) + } +} + +// A disabled hook is not a candidate. +func TestMatcherDisabledHookNotCandidate(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + f.pool.Exec(ctx, `UPDATE hook SET enabled = false WHERE id = $1`, hookID) + + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.count != 0 { + t.Errorf("disabled hook produced %d executions, want 0", r.count) + } +} + +// --------------------------------------------------------------------------- +// Deterministic race regressions for the four matcher must-fixes +// (MUL-4332 PR3 review round @ c4cfdcac8). +// --------------------------------------------------------------------------- + +// Must-fix 1 — the decision is ONE authoritative transaction, so candidate +// revisions are pinned from a single snapshot together with the finalize. +// +// Previously each candidate ran in its own transaction, committing as it went while +// the event stayed un-finalized. That is what let a decision be assembled from +// revisions read at different instants (and, across a retry, mixed). This locks the +// SECOND candidate's hook row and proves the FIRST candidate's decision is still +// invisible while the matcher blocks on it — nothing is committed until the whole +// event is. Under the old per-candidate transactions hook A's execution was already +// durable at this point and this test fails. +func TestMatcherDecisionIsAtomicAcrossCandidates(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + + hookA := f.seedHookAged(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent, "2 minutes") + hookB := f.seedHookAged(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent, "1 minute") + event := f.seedEvent(t, "done", 0) + lease := f.claimWithLease(t, event.ID) + + // Hold hook B's row exactly as a concurrent PATCH (which locks the hook to + // allocate a revision) would. + lockConn, err := f.pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + released := false + release := func() { + if !released { + released = true + lockConn.Release() + } + } + defer release() + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM hook WHERE id = $1 FOR UPDATE`, hookB); err != nil { + t.Fatalf("lock hook B: %v", err) + } + + type result struct { + dispatched bool + err error + } + done := make(chan result, 1) + go func() { + ok, err := f.decideClaimed(ctx, event, lease) + done <- result{dispatched: ok, err: err} + }() + + // While blocked on hook B, NOTHING may be visible yet — not hook A's decision, + // not the finalize. This is the single-transaction guarantee. + select { + case res := <-done: + t.Fatalf("decision completed (dispatched=%v err=%v) while hook B was locked", res.dispatched, res.err) + case <-time.After(750 * time.Millisecond): + } + if r := f.execFor(t, hookA); r.count != 0 { + t.Fatalf("hook A already has %d execution(s) while the event transaction is still open — "+ + "a candidate decision committed on its own instead of within the event's single snapshot", r.count) + } + if s := f.eventState(t, event.ID); s.status != "dispatching" || s.hasDispatchedAt { + t.Fatalf("event finalized early: %+v", s) + } + + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release hook B lock: %v", err) + } + release() + + select { + case res := <-done: + if res.err != nil { + t.Fatalf("decide: %v", res.err) + } + if !res.dispatched { + t.Fatal("decision did not dispatch the event after the lock released") + } + case <-time.After(5 * time.Second): + t.Fatal("decision never unblocked after the hook lock released") + } + + // Both candidates landed together, each pinned to the revision the snapshot saw. + for _, hookID := range []string{hookA, hookB} { + r := f.execFor(t, hookID) + if r.count != 1 || r.status != hookExecQueued { + t.Errorf("hook %s: count=%d status=%q, want exactly 1 queued", hookID, r.count, r.status) + } + var activeRev string + if err := f.pool.QueryRow(ctx, `SELECT active_revision_id::text FROM hook WHERE id = $1`, hookID).Scan(&activeRev); err != nil { + t.Fatal(err) + } + if r.revisionID != activeRev { + t.Errorf("hook %s: execution pinned revision %s, want %s", hookID, r.revisionID, activeRev) + } + } + if s := f.eventState(t, event.ID); s.status != "dispatched" || !s.hasDispatchedAt { + t.Errorf("event state = %+v, want dispatched with dispatched_at set", s) + } +} + +// Must-fix 2 — a stale lease holder must fail closed BEFORE writing anything, and +// the finalize CAS result must be checked. +// +// Previously the lease was only CAS'd on the final UPDATE: a matcher whose lease had +// been reclaimed still wrote its hook_execution and advanced the latch, the CAS then +// affected 0 rows, and the event was still counted as dispatched. Now ownership is +// re-asserted under a row lock first, so the stale holder writes nothing at all. +// This also pins the dispatched_at gap: 'dispatched' must never mean a NULL timestamp. +func TestMatcherStaleLeaseHolderWritesNothing(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + + // rising_edge + a satisfied condition, so a stale write would leave BOTH an + // execution row and an advanced latch behind. + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + f.setIssueStatus(t, "done") + event := f.seedEvent(t, "done", 0) + + owner := f.claimWithLease(t, event.ID) // the lease that currently owns the event + stale := util.NewUUID() // a matcher whose lease was already reclaimed + + // Ownership must be re-asserted BEFORE any decision work, not merely discovered + // at the finalize CAS. Hold the hook row as a concurrent edit would: a stale + // holder that had already started deciding would block here, so returning + // promptly is the observable proof that it failed closed up front. + lockConn, err := f.pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM hook WHERE id = $1 FOR UPDATE`, hookID); err != nil { + t.Fatalf("lock hook: %v", err) + } + + type staleResult struct { + dispatched bool + err error + } + staleDone := make(chan staleResult, 1) + go func() { + ok, err := f.decideClaimed(ctx, event, stale) + staleDone <- staleResult{dispatched: ok, err: err} + }() + + var res staleResult + select { + case res = <-staleDone: + case <-time.After(750 * time.Millisecond): + lockTx.Rollback(ctx) + lockConn.Release() + t.Fatal("stale lease holder blocked on the hook row — it began deciding before re-asserting lease ownership") + } + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release hook lock: %v", err) + } + lockConn.Release() + + if res.err != nil { + t.Fatalf("stale holder returned an error: %v", res.err) + } + if res.dispatched { + t.Error("stale lease holder reported the event dispatched") + } + if r := f.execFor(t, hookID); r.count != 0 { + t.Errorf("stale lease holder wrote %d execution(s), want 0", r.count) + } + if _, found := f.latchFor(t, hookID); found { + t.Error("stale lease holder advanced the rising-edge latch") + } + if s := f.eventState(t, event.ID); s.status != "dispatching" || s.lease != util.UUIDToString(owner) || s.hasDispatchedAt { + t.Errorf("stale holder disturbed the event: %+v", s) + } + + // The real owner still decides it normally. + ok, err := f.decideClaimed(ctx, event, owner) + if err != nil { + t.Fatalf("owner: %v", err) + } + if !ok { + t.Fatal("owner did not dispatch the event") + } + if r := f.execFor(t, hookID); r.count != 1 || r.status != hookExecQueued { + t.Errorf("owner decision: count=%d status=%q, want 1 queued", r.count, r.status) + } + s := f.eventState(t, event.ID) + if s.status != "dispatched" { + t.Errorf("event status = %q, want dispatched", s.status) + } + if !s.hasDispatchedAt { + t.Error("dispatched event has a NULL dispatched_at — the retention/audit boundary cannot rely on it") + } + if s.lease != "" { + t.Errorf("dispatched event still holds lease %s", s.lease) + } +} + +// Must-fix 3 — the depth guard decides only whether THIS event fires; it must not +// skip the rising-edge latch advance. +// +// Elon's sequence: true → (an over-deep event observes false) → true. Previously the +// guard returned before touching the latch, so the middle event never reset it and +// the next legitimate false→true edge was swallowed as condition_already_true. +func TestMatcherDepthGuardStillAdvancesLatch(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + + assertLast := func(step, wantStatus, wantReason string) { + t.Helper() + if r := f.execFor(t, hookID); r.status != wantStatus || r.skipReason != wantReason { + t.Fatalf("%s: want status=%q reason=%q, got %+v", step, wantStatus, wantReason, r) + } + } + + // 1. false→true at a normal depth: fires, latch true. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast("initial edge", hookExecQueued, "") + if satisfied, found := f.latchFor(t, hookID); !found || !satisfied { + t.Fatalf("latch after initial edge: satisfied=%v found=%v, want true/true", satisfied, found) + } + + // 2. The condition drops to false and is observed by an OVER-DEEP event. The + // guard skips the fire, but the observation must still be recorded. + f.setIssueStatus(t, "todo") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "todo", maxHopCount+1)); err != nil { + t.Fatal(err) + } + assertLast("over-deep event", hookExecSkipped, skipMaxDepth) + if satisfied, found := f.latchFor(t, hookID); !found || satisfied { + t.Fatalf("the depth guard skipped the latch advance: satisfied=%v found=%v, want false/true — "+ + "a matched event must record its condition state even when the guard rejects the fire", satisfied, found) + } + + // 3. The next legitimate false→true edge must fire, not be swallowed. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast("edge after the guarded event", hookExecQueued, "") +} + +// Must-fix 4 — one unusable revision must not starve the healthy rules on the same +// event, and must not make the event retry forever. +// +// Previously MatchEvent returned at the first error, so a poison candidate early in +// the fixed candidate order meant later healthy hooks got no execution and the event +// was never finalized — it re-leased and re-hit the same poison on every tick. +func TestMatcherPoisonHookDoesNotStarveHealthy(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + + // `conditions` must be a JSON array; an object is valid jsonb the evaluator can + // never interpret — a deterministic config failure, not a transient one. + poison := f.seedHookAged(t, "issue.status_changed", `{"to":"done"}`, `{"broken":true}`, automation.FirePerEvent, "2 minutes") + healthy := f.seedHookAged(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent, "1 minute") + event := f.seedEvent(t, "done", 0) + lease := f.claimWithLease(t, event.ID) + + ok, err := f.decideClaimed(ctx, event, lease) + if err != nil { + t.Fatalf("decide: %v", err) + } + if !ok { + t.Fatal("event was not dispatched — one poison candidate blocked the whole event") + } + + if r := f.execFor(t, healthy); r.count != 1 || r.status != hookExecQueued { + t.Errorf("healthy hook: count=%d status=%q, want 1 queued — a poison candidate starved it", r.count, r.status) + } + r := f.execFor(t, poison) + if r.count != 1 || r.status != hookExecFailed { + t.Errorf("poison hook: count=%d status=%q, want exactly 1 failed isolation row", r.count, r.status) + } + if r.errorCode != errCodeInvalidConfig { + t.Errorf("poison hook error_code = %q, want %q", r.errorCode, errCodeInvalidConfig) + } + if s := f.eventState(t, event.ID); s.status != "dispatched" || !s.hasDispatchedAt { + t.Errorf("event state = %+v, want dispatched with dispatched_at set", s) + } + + // Re-deciding the same event stays idempotent for both candidates. + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("re-decide: %v", err) + } + if r := f.execFor(t, healthy); r.count != 1 { + t.Errorf("healthy hook has %d executions after re-decide, want 1", r.count) + } + if r := f.execFor(t, poison); r.count != 1 { + t.Errorf("poison hook has %d executions after re-decide, want 1", r.count) + } +} + +// --------------------------------------------------------------------------- +// Regressions for the second matcher review round: claim-time revision pin under +// a single snapshot, and zero writes from an expired lease. +// --------------------------------------------------------------------------- + +// drainMatcherQueue empties the shared outbox so a following controlled claim lands +// on this test's own event. +func (f matcherFixture) drainMatcherQueue(t *testing.T) { + t.Helper() + ctx := context.Background() + for i := 0; i < 60; i++ { + n, err := f.svc.ClaimAndMatch(ctx, 500) + if err != nil { + t.Fatalf("drain: %v", err) + } + if n == 0 { + return + } + } + t.Fatal("outbox never drained") +} + +// sleepDuringClaim makes the matcher's claim UPDATE on one specific event pause, so +// a test can commit a concurrent edit inside the claim window. +func (f matcherFixture) sleepDuringClaim(t *testing.T, eventID pgtype.UUID, seconds float64) { + t.Helper() + ctx := context.Background() + name := fmt.Sprintf("hook_claim_sleep_%d", time.Now().UnixNano()) + fn := name + "_fn" + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN PERFORM pg_sleep(%f); RETURN NEW; END; $$;`, quoteIdent(fn), seconds)); err != nil { + t.Fatalf("create claim sleep function: %v", err) + } + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE TRIGGER %s BEFORE UPDATE ON domain_event + FOR EACH ROW WHEN (NEW.id = %s::uuid AND NEW.dispatch_status = 'dispatching') + EXECUTE FUNCTION %s();`, quoteIdent(name), quoteLiteral(util.UUIDToString(eventID)), quoteIdent(fn))); err != nil { + t.Fatalf("create claim sleep trigger: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, fmt.Sprintf("DROP TRIGGER IF EXISTS %s ON domain_event", quoteIdent(name))) + f.pool.Exec(bg, fmt.Sprintf("DROP FUNCTION IF EXISTS %s()", quoteIdent(fn))) + }) +} + +// Must-fix 1 (round 2) — the revision set is pinned by the CLAIM statement itself. +// +// Wrapping the decision in a transaction is not enough: PostgreSQL's default READ +// COMMITTED gives every statement a fresh snapshot, so claiming in one statement and +// resolving candidates in another lets an edit committed in between change the +// decision — and lets two candidates in one transaction be decided against revisions +// from different instants. This drives the real ClaimAndMatch, pauses inside the +// claim, and commits a revision swap to a DIFFERENT event type in that window. The +// claim and the candidate set share one snapshot, so the decision must still bind +// revision 1. Anything that resolved candidates after the claim would see revision 2, +// find no candidate, and write nothing. +func TestMatcherPinsRevisionAtClaimTime(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.drainMatcherQueue(t) + + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + var rev1 string + if err := f.pool.QueryRow(ctx, `SELECT active_revision_id::text FROM hook WHERE id = $1`, hookID).Scan(&rev1); err != nil { + t.Fatal(err) + } + event := f.seedEvent(t, "done", 0) + f.sleepDuringClaim(t, event.ID, 0.8) + + done := make(chan error, 1) + go func() { + _, err := f.svc.ClaimAndMatch(ctx, 5) + done <- err + }() + + // Land the edit inside the claim window: a new revision listening to a different + // event type, made active while the claim is still executing. + time.Sleep(250 * time.Millisecond) + rev2 := uuid.NewString() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook_revision (id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id) + VALUES ($1, $2, 2, 'issue.created', '{}'::jsonb, '[]'::jsonb, $3, '[]'::jsonb, 'member', $4)`, + rev2, hookID, automation.FirePerEvent, f.userID); err != nil { + t.Fatalf("seed revision 2: %v", err) + } + if _, err := f.pool.Exec(ctx, `UPDATE hook SET active_revision_id = $2 WHERE id = $1`, hookID, rev2); err != nil { + t.Fatalf("repoint active revision: %v", err) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("claim and match: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("claim and match never returned") + } + + r := f.execFor(t, hookID) + if r.count != 1 || r.status != hookExecQueued { + t.Fatalf("want exactly 1 queued execution from the claim-time revision, got count=%d status=%q "+ + "(a revision edit landing after the claim changed the decision)", r.count, r.status) + } + if r.revisionID != rev1 { + t.Errorf("execution pinned revision %s, want the claim-time revision %s", r.revisionID, rev1) + } + if s := f.eventState(t, event.ID); s.status != "dispatched" || !s.hasDispatchedAt { + t.Errorf("event state = %+v, want dispatched with dispatched_at set", s) + } +} + +// Must-fix 2 (round 2) — a worker whose lease has EXPIRED writes nothing. +// +// The ownership predicate previously checked only status + token, so a worker still +// holding the right token past its expiry could decide and finalize. Ownership now +// requires the lease to be unexpired under database clock time, at entry and at the +// finalize CAS alike. Driving the real ClaimAndMatch with an already-elapsed TTL +// makes the claimed lease expired on arrival, so the whole transaction must roll back. +func TestMatcherExpiredLeaseWritesNothing(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.drainMatcherQueue(t) + + // rising_edge with a satisfied condition, so a leaked write would leave BOTH an + // execution row and an advanced latch behind. + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + f.setIssueStatus(t, "done") + event := f.seedEvent(t, "done", 0) + + original := MatcherLeaseTTL + t.Cleanup(func() { MatcherLeaseTTL = original }) + + // Every lease this tick acquires is already expired when it is granted. Hold the + // hook row as a concurrent edit would: an expired worker must fail closed BEFORE + // any decision work, so it never reaches (and never blocks on) this lock. + MatcherLeaseTTL = -1 * time.Minute + lockConn, err := f.pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM hook WHERE id = $1 FOR UPDATE`, hookID); err != nil { + t.Fatalf("lock hook: %v", err) + } + + expired := make(chan error, 1) + go func() { + _, err := f.svc.ClaimAndMatch(ctx, 10) + expired <- err + }() + select { + case err := <-expired: + if err != nil { + t.Fatalf("claim and match with an expired lease: %v", err) + } + case <-time.After(1 * time.Second): + lockTx.Rollback(ctx) + lockConn.Release() + t.Fatal("expired-lease worker blocked on the hook row — it began deciding before asserting ownership") + } + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release hook lock: %v", err) + } + lockConn.Release() + + if r := f.execFor(t, hookID); r.count != 0 { + t.Errorf("expired-lease worker wrote %d execution(s), want 0", r.count) + } + if _, found := f.latchFor(t, hookID); found { + t.Error("expired-lease worker advanced the rising-edge latch") + } + if s := f.eventState(t, event.ID); s.status == "dispatched" || s.hasDispatchedAt { + t.Errorf("expired-lease worker finalized the event: %+v", s) + } + + // Losing ownership now also backs the event off (see + // TestMatcherExpiredLeaseEventDoesNotStarveQueue), so it is not immediately + // claimable. Advance past that backoff to reach the retry. + if _, err := f.pool.Exec(ctx, + `UPDATE domain_event SET available_at = now() WHERE id = $1`, + util.UUIDToString(event.ID)); err != nil { + t.Fatalf("clear backoff: %v", err) + } + + // With a live lease the same event decides normally. + MatcherLeaseTTL = original + if _, err := f.svc.ClaimAndMatch(ctx, 10); err != nil { + t.Fatalf("claim and match: %v", err) + } + if r := f.execFor(t, hookID); r.count != 1 || r.status != hookExecQueued { + t.Errorf("live-lease decision: count=%d status=%q, want 1 queued", r.count, r.status) + } + if s := f.eventState(t, event.ID); s.status != "dispatched" || !s.hasDispatchedAt { + t.Errorf("event state = %+v, want dispatched with dispatched_at set", s) + } +} + +// The finalize CAS carries the same not-expired condition as the entry assertion, so +// a lease that elapses mid-decision cannot commit the decision it already wrote. +func TestMatcherFinalizeRejectsExpiredLease(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + event := f.seedEvent(t, "done", 0) + lease := f.claimWithLease(t, event.ID) + + // The lease elapses while the decision is in flight. + if _, err := f.pool.Exec(ctx, + `UPDATE domain_event SET lease_expires_at = now() - interval '1 second' WHERE id = $1`, + util.UUIDToString(event.ID)); err != nil { + t.Fatalf("expire lease: %v", err) + } + + rows, err := f.svc.Queries.MarkDomainEventDispatched(ctx, db.MarkDomainEventDispatchedParams{ + ID: event.ID, LeaseToken: lease, + }) + if err != nil { + t.Fatalf("finalize: %v", err) + } + if rows != 0 { + t.Errorf("finalize affected %d row(s) for an expired lease, want 0", rows) + } + if s := f.eventState(t, event.ID); s.status == "dispatched" || s.hasDispatchedAt { + t.Errorf("expired lease finalized the event: %+v", s) + } +} + +// sleepBeforeExecutionInsert makes every hook_execution INSERT for one hook pause, +// so a test can make a decision reliably outlive its lease TTL. +func (f matcherFixture) sleepBeforeExecutionInsert(t *testing.T, hookID string, seconds float64) { + t.Helper() + ctx := context.Background() + name := fmt.Sprintf("hook_exec_sleep_%d", time.Now().UnixNano()) + fn := name + "_fn" + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN PERFORM pg_sleep(%f); RETURN NEW; END; $$;`, quoteIdent(fn), seconds)); err != nil { + t.Fatalf("create execution sleep function: %v", err) + } + if _, err := f.pool.Exec(ctx, fmt.Sprintf(` + CREATE TRIGGER %s BEFORE INSERT ON hook_execution + FOR EACH ROW WHEN (NEW.hook_id = %s::uuid) + EXECUTE FUNCTION %s();`, quoteIdent(name), quoteLiteral(hookID), quoteIdent(fn))); err != nil { + t.Fatalf("create execution sleep trigger: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, fmt.Sprintf("DROP TRIGGER IF EXISTS %s ON hook_execution", quoteIdent(name))) + f.pool.Exec(bg, fmt.Sprintf("DROP FUNCTION IF EXISTS %s()", quoteIdent(fn))) + }) +} + +// availableInFuture reports whether an event has been backed off past now. +func (f matcherFixture) availableInFuture(t *testing.T, eventID pgtype.UUID) bool { + t.Helper() + var future bool + if err := f.pool.QueryRow(context.Background(), + `SELECT available_at > now() FROM domain_event WHERE id = $1`, + util.UUIDToString(eventID)).Scan(&future); err != nil { + t.Fatalf("read available_at: %v", err) + } + return future +} + +// An event whose OWN fresh lease expires mid-decision must be backed off like any +// other failed decision, so the queue moves past it. +// +// Zero writes alone is not enough. Claim and decision share one transaction and the +// event row stays locked throughout, so in production this branch is essentially +// always "the lease we just took expired while we were deciding". Rolling back also +// undoes the claim and restores the original available_at, so a decision that +// reliably outlives its TTL would be re-selected as the oldest event on every tick +// and starve everything behind it forever. This puts such an event at the head of +// the queue with a healthy event behind it and asserts the healthy one still runs. +func TestMatcherExpiredLeaseEventDoesNotStarveQueue(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.drainMatcherQueue(t) + + // The head event matches this hook, and writing its execution always takes + // longer than the lease TTL below — so ownership is lost every single attempt. + stuckHook := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + f.sleepBeforeExecutionInsert(t, stuckHook, 0.7) + + head := f.seedEvent(t, "done", 0) // matches stuckHook → slow execution insert + behind := f.seedEvent(t, "todo", 0) // no candidate matches → decides instantly + if head.Seq >= behind.Seq { + t.Fatalf("head event must be older: head seq %d, behind seq %d", head.Seq, behind.Seq) + } + + original := MatcherLeaseTTL + t.Cleanup(func() { MatcherLeaseTTL = original }) + MatcherLeaseTTL = 300 * time.Millisecond + + // Two ticks, mirroring a running matcher. + for round := 1; round <= 2; round++ { + if _, err := f.svc.ClaimAndMatch(ctx, 10); err != nil { + t.Fatalf("round %d: %v", round, err) + } + } + + // The stuck event stayed safe: no partial decision survived. + if r := f.execFor(t, stuckHook); r.count != 0 { + t.Errorf("expired-lease event wrote %d execution(s), want 0", r.count) + } + if s := f.eventState(t, head.ID); s.status == "dispatched" || s.hasDispatchedAt { + t.Errorf("expired-lease event was finalized: %+v", s) + } + + // ...and it was backed off rather than left holding the head of the queue. + if !f.availableInFuture(t, head.ID) { + t.Error("expired-lease event was not backed off — it stays the oldest claimable " + + "event and will be re-selected on every tick") + } + + // The decisive assertion: the queue moved past it. + if s := f.eventState(t, behind.ID); s.status != "dispatched" || !s.hasDispatchedAt { + t.Errorf("event behind the stuck one = %+v, want dispatched — one event whose lease "+ + "keeps expiring is starving the whole queue", s) + } +} diff --git a/server/internal/service/issue.go b/server/internal/service/issue.go index 124d8c5ab93..6b8257f3e40 100644 --- a/server/internal/service/issue.go +++ b/server/internal/service/issue.go @@ -9,6 +9,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/issueposition" @@ -315,6 +316,13 @@ func (s *IssueService) Create(ctx context.Context, p IssueCreateParams, opts Iss } } + // Transactional outbox (MUL-4332): the issue.created fact and its domain + // event commit atomically. The payload carries the birth assignee so a hook + // can react to assignment-at-create without a separate issue.assigned event. + if _, err := domainevent.Write(ctx, qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + return IssueCreateResult{}, fmt.Errorf("write issue.created event: %w", err) + } + if err := tx.Commit(ctx); err != nil { return IssueCreateResult{}, fmt.Errorf("commit: %w", err) } diff --git a/server/internal/service/recover_orphans_cursor_test.go b/server/internal/service/recover_orphans_cursor_test.go new file mode 100644 index 00000000000..49314dc71e7 --- /dev/null +++ b/server/internal/service/recover_orphans_cursor_test.go @@ -0,0 +1,134 @@ +package service + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The recover-orphans drain must step PAST a page made entirely of poison +// (unresolvable-workspace) rows to reach the healthy rows behind them (MUL-4332 +// review round 3, point 1). A poison row is skipped rather than failed, so it stays +// selectable; without the keyset cursor a plain "re-select the oldest N" drain would +// keep re-reading that same poison page forever and never fail the healthy rows +// behind it. The cursor advances over every candidate the page locked — poison +// included — so the next page moves on. +// +// A real agent_task_queue row always resolves its workspace via its agent, so (as in +// TestFailBulkTasksIsolatesPoisonRow) we present the two oldest rows to the fail path +// as poison by stripping their resolvable links in-memory; their id/created_at stay +// intact so the cursor still advances over them, and the DB rows are left untouched +// and un-failed, exactly as a genuinely unresolvable row would be. +func TestRecoverOrphansCursorStepsPastPoisonPage(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var runtimeID string + if err := pool.QueryRow(ctx, `SELECT runtime_id FROM agent WHERE id = $1`, agentID).Scan(&runtimeID); err != nil { + t.Fatalf("load runtime id: %v", err) + } + + // Three orphans on the runtime with DISTINCT, increasing created_at so ordering + // is by created_at (exercising the keyset created_at branch, complementing the + // id-tiebreaker path). The two OLDEST are the poison page; the newest is healthy. + seed := func(ageInterval string) string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, created_at) + VALUES ($1, $2, $3, 'running', 0, now() - $4::interval) + RETURNING id`, agentID, runtimeID, issueID, ageInterval).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + poison1 := seed("3 minutes") + poison2 := seed("2 minutes") + healthy := seed("1 minute") + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, []string{poison1, poison2, healthy}) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, []string{poison1, poison2, healthy}) + }) + isPoison := map[string]bool{poison1: true, poison2: true} + + // Drain exactly as the handler does, but with a page size of 2 so the two poison + // rows fill page 1 completely. If the cursor did not advance past them, page 2 + // would re-select the same poison page and the healthy row would never fail. + const pageSize = 2 + var afterCreatedAt pgtype.Timestamptz + var afterID pgtype.UUID + drained := 0 + for page := 0; page < 5; page++ { + var candidates []db.AgentTaskQueue + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: util.MustParseUUID(runtimeID), + AfterCreatedAt: afterCreatedAt, + AfterID: afterID, + MaxPerTick: pageSize, + }) + if e != nil { + return nil, e + } + // Strip links on the poison rows so the fail path cannot resolve a + // workspace and skips them (leaving them 'running'). id/created_at are + // untouched, so the cursor still advances over them. + for i := range c { + if isPoison[util.UUIDToString(c[i].ID)] { + c[i].AgentID = pgtype.UUID{} + c[i].IssueID = pgtype.UUID{} + c[i].ChatSessionID = pgtype.UUID{} + c[i].AutopilotRunID = pgtype.UUID{} + } + } + candidates = c + return c, e + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("page %d: FailBulkTasksWithEvents: %v", page, err) + } + drained += len(failed) + if len(candidates) < pageSize { + break // short page → drained + } + last := candidates[len(candidates)-1] + afterCreatedAt = last.CreatedAt + afterID = last.ID + } + + // The healthy row behind the poison page got failed with its event... + if s := taskStatusForTest(t, pool, healthy); s != "failed" { + t.Errorf("healthy row status = %q, want failed (cursor must step past the poison page)", s) + } + if n := subjectEventCount(t, pool, healthy); n != 1 { + t.Errorf("healthy row events = %d, want 1", n) + } + // ...and the poison rows were left untouched — skipped, never failed, no event. + for _, pid := range []string{poison1, poison2} { + if s := taskStatusForTest(t, pool, pid); s != "running" { + t.Errorf("poison row %s status = %q, want running (skipped, not failed)", pid, s) + } + if n := subjectEventCount(t, pool, pid); n != 0 { + t.Errorf("poison row %s events = %d, want 0", pid, n) + } + } + if drained != 1 { + t.Errorf("drained = %d, want 1 (only the healthy row behind the poison page)", drained) + } +} diff --git a/server/internal/service/recover_orphans_lock_wait_test.go b/server/internal/service/recover_orphans_lock_wait_test.go new file mode 100644 index 00000000000..4ed9719019d --- /dev/null +++ b/server/internal/service/recover_orphans_lock_wait_test.go @@ -0,0 +1,168 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The recover-orphans query pages forward with a keyset cursor that advances +// permanently past whatever a page returned, so it MUST NOT use FOR UPDATE SKIP +// LOCKED (MUL-4332 review round 4, point 1). If an older orphan is briefly locked +// while a page is being selected — a sweep or a stale-dispatch reclaim holding it — +// SKIP LOCKED would silently drop it; the page, filled by the newer rows behind it, +// would advance the cursor past that older row, and no later page could ever select +// it again (the runtime is already back `online`, so the offline sweep won't reap it +// either). Plain FOR UPDATE instead WAITS for the lock and, once it releases, +// re-checks the row against the WHERE and includes it. +// +// This reproduces exactly that race: lock the OLDEST orphan in a concurrent +// transaction, prove the drain BLOCKS (does not skip past) while the lock is held, +// then release it and assert the previously-locked row is failed with its event. +// Under the old SKIP LOCKED query the drain would finish immediately with the oldest +// row still `running` forever, and this test would fail. +func TestRecoverOrphansWaitsForLockedOldestOrphan(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var runtimeID string + if err := pool.QueryRow(ctx, `SELECT runtime_id FROM agent WHERE id = $1`, agentID).Scan(&runtimeID); err != nil { + t.Fatalf("load runtime id: %v", err) + } + + // Three healthy orphans with distinct, increasing created_at. The OLDEST is the + // one we lock while the page is selected. + seed := func(ageInterval string) string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, created_at) + VALUES ($1, $2, $3, 'running', 0, now() - $4::interval) + RETURNING id`, agentID, runtimeID, issueID, ageInterval).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + oldest := seed("3 minutes") + mid := seed("2 minutes") + newest := seed("1 minute") + all := []string{oldest, mid, newest} + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, all) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, all) + }) + + // Hold a FOR UPDATE lock on the oldest orphan in a dedicated connection, exactly + // as a sweep or a stale-dispatch reclaim would briefly hold it. + lockConn, err := pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + lockReleased := false + releaseLock := func() { + if lockReleased { + return + } + lockReleased = true + lockConn.Release() + } + defer releaseLock() + + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM agent_task_queue WHERE id = $1 FOR UPDATE`, oldest); err != nil { + t.Fatalf("lock oldest orphan: %v", err) + } + + // Run one drain page in the background. A page size above the row count means a + // single page covers all three — so blocking on the oldest row blocks the whole + // page (nothing is partially failed until the lock releases). + type drainResult struct { + failed int + err error + } + done := make(chan drainResult, 1) + go func() { + var candidates []db.AgentTaskQueue + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: util.MustParseUUID(runtimeID), + MaxPerTick: 500, + }) + candidates = c + return c, e + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) + _ = candidates + done <- drainResult{failed: len(failed), err: err} + }() + + // While the lock is held the drain must NOT finish: plain FOR UPDATE blocks on + // the locked oldest row instead of skipping it. + select { + case res := <-done: + if res.err != nil { + t.Fatalf("drain errored while the oldest orphan was locked: %v", res.err) + } + t.Fatalf("drain completed (%d failed) while the oldest orphan was locked — it skipped instead of waiting (SKIP LOCKED regression)", res.failed) + case <-time.After(750 * time.Millisecond): + // Expected: blocked on the lock. + } + + // Nothing may be partially failed while the page is blocked behind the lock. + for _, id := range all { + if s := taskStatusForTest(t, pool, id); s != "running" { + t.Fatalf("row %s = %q before unlock, want running (a blocked page must fail nothing yet)", id, s) + } + } + + // Release the lock; the drain must now unblock and fail every orphan. + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release lock: %v", err) + } + releaseLock() + + select { + case res := <-done: + if res.err != nil { + t.Fatalf("drain errored after the lock released: %v", res.err) + } + if res.failed != len(all) { + t.Errorf("failed = %d, want %d (the drain must include the previously-locked oldest row)", res.failed, len(all)) + } + case <-time.After(5 * time.Second): + t.Fatal("drain did not complete after the lock released — plain FOR UPDATE never unblocked") + } + + // The previously-locked oldest orphan is now failed with exactly one event. + if s := taskStatusForTest(t, pool, oldest); s != "failed" { + t.Errorf("oldest orphan status = %q, want failed (plain FOR UPDATE must wait for the lock, not skip past)", s) + } + if n := subjectEventCount(t, pool, oldest); n != 1 { + t.Errorf("oldest orphan events = %d, want 1 (fact ⇔ event for the recovered row)", n) + } + // The other two rows are failed too, so the whole page landed. + for _, id := range []string{mid, newest} { + if s := taskStatusForTest(t, pool, id); s != "failed" { + t.Errorf("row %s status = %q, want failed", id, s) + } + } +} diff --git a/server/internal/service/task.go b/server/internal/service/task.go index 377856b185d..af5f73b7b83 100644 --- a/server/internal/service/task.go +++ b/server/internal/service/task.go @@ -16,6 +16,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" "github.com/multica-ai/multica/server/internal/attribution" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/featureflags" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" @@ -2625,6 +2626,25 @@ func (s *TaskService) CompleteTask(ctx context.Context, taskID pgtype.UUID, resu } chatAssistantMsg = msg } + + // Transactional outbox (MUL-4332): emit task.completed atomically with + // the status flip. The status CAS above makes this exactly-once — a + // replayed callback finds the task already terminal and never re-emits. + // Fail-closed on workspace resolution (review point 4): if the workspace + // is unresolvable we error out and roll the completion back rather than + // commit the fact without its event. + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + return fmt.Errorf("task.completed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + } + evt := domainevent.TaskCompleted(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskCompletedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.completed event: %w", err) + } return nil }); err != nil { // When parallel agents race, a task may already be completed, @@ -2953,6 +2973,27 @@ func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, } retried = &child } + + // Transactional outbox (MUL-4332): emit task.failed atomically with the + // fail (and any retry child). retry_eligible is the atomically-decidable + // eligibility predicate — NOT a promise a child was created — computed from + // the just-failed row so it matches the bulk paths exactly; here a child is + // in fact created in this same tx when eligible. The status CAS makes it + // exactly-once. Fail-closed on workspace resolution (review point 4). + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + return fmt.Errorf("task.failed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + } + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskFailedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + RetryEligible: retryEligible(failureReason, t), + ErrorCode: failureReason, + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.failed event: %w", err) + } return nil }); err != nil { if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil { @@ -3431,23 +3472,61 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas "error", checkErr, ) } else if !hasActive { - updatedIssue, updateErr := s.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ - ID: t.IssueID, - Status: "todo", - WorkspaceID: issue.WorkspaceID, + var updatedIssue db.Issue + var didReset bool + // Transactional outbox (MUL-4332): the stuck-issue reset bypasses the + // HTTP UpdateIssue path, so emit issue.status_changed atomically here. + updateErr := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + // Re-decide the reset UNDER the row lock (review point 4). The + // in_progress + no-active-task check that gated this branch was read + // outside the tx, so within the window a user could have moved the + // issue to done, or a fresh task could have been claimed. Only a + // still-in_progress issue with still no active task is reset — a + // user-completed issue must never be silently reopened. `from` then + // always reflects the true edge (also review point 3). + before, bErr := qtx.LockIssueRowForUpdate(ctx, db.LockIssueRowForUpdateParams{ + ID: t.IssueID, + WorkspaceID: issue.WorkspaceID, + }) + if bErr != nil { + return nil, bErr + } + if before.Status != "in_progress" { + return nil, nil + } + stillActive, aErr := qtx.HasActiveTaskForIssue(ctx, t.IssueID) + if aErr != nil { + return nil, aErr + } + if stillActive { + return nil, nil + } + u, uErr := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ + ID: t.IssueID, + Status: "todo", + WorkspaceID: issue.WorkspaceID, + }) + if uErr != nil { + return nil, uErr + } + updatedIssue = u + didReset = true + return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), + domainevent.IssueStatusChangedPayload{From: before.Status, To: u.Status})}, nil }) if updateErr != nil { slog.Warn("handle failed tasks: reset stuck issue failed", "issue_id", issueKey, "error", updateErr, ) - } else { + } else if didReset { // This direct reset bypasses the HTTP UpdateIssue // handler that normally emits issue:updated, so emit // it here too. Without it the board / status-filter // caches keep showing the issue as in_progress until - // the next write touches it (#4648 / MUL-3782). - s.broadcastIssueUpdated(updatedIssue, issue.Status) + // the next write touches it (#4648 / MUL-3782). The reset + // only fires from in_progress, so that is the true prev. + s.broadcastIssueUpdated(updatedIssue, "in_progress") } } } @@ -3485,6 +3564,152 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas // runInTx executes fn inside a single DB transaction. If TxStarter is nil // (e.g. some tests construct TaskService directly), fn runs against the // regular Queries handle without transactional guarantees. +// resolveTaskWorkspaceForEvent resolves a task's workspace for stamping a task +// domain event (agent_task_queue carries no workspace column). It reads on the +// passed qtx so the lookup stays in the caller's transaction, and walks the +// task's stable attribution — issue, chat session, autopilot run, quick-create +// context — before finally falling back to the owning agent. This mirrors +// ResolveTaskWorkspaceID's chain but tx-scoped and with the agent fallback that +// resolver lacks. An invalid return means genuinely unresolvable; the caller +// treats that as fail-closed (review point 4): it errors out and rolls the whole +// terminal transition back rather than committing a fact with no event. +func (s *TaskService) resolveTaskWorkspaceForEvent(ctx context.Context, qtx *db.Queries, task db.AgentTaskQueue) pgtype.UUID { + if task.IssueID.Valid { + if issue, err := qtx.GetIssue(ctx, task.IssueID); err == nil { + return issue.WorkspaceID + } + } + if task.ChatSessionID.Valid { + if cs, err := qtx.GetChatSession(ctx, task.ChatSessionID); err == nil { + return cs.WorkspaceID + } + } + if task.AutopilotRunID.Valid { + if run, err := qtx.GetAutopilotRun(ctx, task.AutopilotRunID); err == nil { + if ap, err := qtx.GetAutopilot(ctx, run.AutopilotID); err == nil { + return ap.WorkspaceID + } + } + } + if qc, ok := s.parseQuickCreateContext(task); ok { + if ws, err := util.ParseUUID(qc.WorkspaceID); err == nil && ws.Valid { + return ws + } + } + if task.AgentID.Valid { + if agent, err := qtx.GetAgent(ctx, task.AgentID); err == nil { + return agent.WorkspaceID + } + } + return pgtype.UUID{} +} + +// FailBulkTasksWithEvents is the poison-isolated bulk fail path (MUL-4332 review +// points 2 & 3). Rather than one condition-scoped UPDATE that fails every match +// in a single statement — where one corrupt row rolls back the whole batch and +// an unbounded match can lock a huge span — it runs, in ONE transaction: +// +// 1. selectFn row-locks a BOUNDED candidate batch (FOR UPDATE SKIP LOCKED + +// LIMIT), so the lock hold is short and we never contend with a daemon's +// claim path; +// 2. each candidate's workspace is resolved on the same tx. A task whose +// workspace is genuinely unresolvable (corrupt / orphaned historical row) +// is EXCLUDED and logged — it can never have a valid event, so failing it +// would strand a fact without one; skipping it fail-closed keeps the healthy +// tasks in the batch unblocked (review point 2); +// 3. failFn fails only the resolvable id set; +// 4. one task.failed event is written per failed row, atomically with the fail. +// +// The actor is the platform (SystemActor): these are sweeper / orphan-recovery +// paths, not an agent action — the agent is already carried in the payload +// (review point 3). retry_eligible is the same atomically-decidable predicate +// HandleFailedTasks uses to gate the post-commit auto-retry, committed in this +// transaction. It reports eligibility only: on these paths the retry child is +// created best-effort AFTER commit, so the event never promises a fresh attempt +// will actually arrive (review point 3, third round). +// +// A transient event/commit failure rolls back only the CURRENT batch (no fact is +// ever committed without its event), so the next sweep tick simply re-selects and +// retries — every caller runs on a fixed cadence. +func (s *TaskService) FailBulkTasksWithEvents( + ctx context.Context, + selectFn func(qtx *db.Queries) ([]db.AgentTaskQueue, error), + failFn func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error), +) ([]db.AgentTaskQueue, error) { + tx, err := s.TxStarter.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + qtx := s.Queries.WithTx(tx) + + candidates, err := selectFn(qtx) + if err != nil { + return nil, err + } + if len(candidates) == 0 { + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return nil, nil + } + + ids := make([]pgtype.UUID, 0, len(candidates)) + wsByID := make(map[string]pgtype.UUID, len(candidates)) + for _, t := range candidates { + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + // Fail-closed but ISOLATED: a task with no resolvable workspace can + // have no valid event, so skip it (leaving it for ops to reap) instead + // of aborting the batch and stranding every healthy task with it. + slog.Warn("bulk fail: skipping task with unresolvable workspace", + "task_id", util.UUIDToString(t.ID), + "agent_id", util.UUIDToString(t.AgentID)) + continue + } + ids = append(ids, t.ID) + wsByID[util.UUIDToString(t.ID)] = wsID + } + if len(ids) == 0 { + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return nil, nil + } + + failed, err := failFn(qtx, ids) + if err != nil { + return nil, err + } + for _, t := range failed { + wsID, ok := wsByID[util.UUIDToString(t.ID)] + if !ok || !wsID.Valid { + // Defensive: failFn returned a row we never resolved. Impossible in one + // tx (ids came from wsByID), but fail-closed rather than emit an event + // with no workspace. + return nil, fmt.Errorf("task.failed event: unresolved workspace for task %s", util.UUIDToString(t.ID)) + } + reason := "" + if t.FailureReason.Valid { + reason = t.FailureReason.String + } + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.SystemActor(), + domainevent.TaskFailedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + RetryEligible: retryEligible(reason, t), + ErrorCode: reason, + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return nil, err + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return failed, nil +} + func (s *TaskService) runInTx(ctx context.Context, fn func(*db.Queries) error) error { if s.TxStarter == nil { return fn(s.Queries) @@ -3960,17 +4185,32 @@ func (s *TaskService) createAgentComment(ctx context.Context, issueID, agentID p rootComment = &root } } - comment, err := s.Queries.CreateComment(ctx, db.CreateCommentParams{ - IssueID: issueID, - WorkspaceID: issue.WorkspaceID, - AuthorType: "agent", - AuthorID: agentID, - Content: content, - Type: commentType, - ParentID: parentID, - SourceTaskID: sourceTaskID, - }) - if err != nil { + // Transactional outbox (MUL-4332 review point 2): the agent runtime comment + // and its comment.created event commit in one transaction. + var comment db.Comment + if err := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, cErr := qtx.CreateComment(ctx, db.CreateCommentParams{ + IssueID: issueID, + WorkspaceID: issue.WorkspaceID, + AuthorType: "agent", + AuthorID: agentID, + Content: content, + Type: commentType, + ParentID: parentID, + SourceTaskID: sourceTaskID, + }) + if cErr != nil { + return nil, cErr + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, domainevent.AgentActor(agentID), + domainevent.CommentCreatedPayload{ + IssueID: util.UUIDToString(created.IssueID), + AuthorType: "agent", + AuthorID: util.UUIDToString(agentID), + ParentID: util.UUIDToString(parentID), + })}, nil + }); err != nil { return } s.CancelDeferredEscalationsForIssueAgent(ctx, issueID, agentID) diff --git a/server/internal/util/pgx.go b/server/internal/util/pgx.go index 5b344067c6a..10d7ec1fe09 100644 --- a/server/internal/util/pgx.go +++ b/server/internal/util/pgx.go @@ -5,9 +5,17 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +// NewUUID mints a fresh random (v4) pgtype.UUID. Use for app-generated primary +// keys — e.g. two related rows that must reference each other before either is +// inserted, so neither can rely on a DB DEFAULT round-trip. +func NewUUID() pgtype.UUID { + return pgtype.UUID{Bytes: uuid.New(), Valid: true} +} + // ParseUUID parses s into a pgtype.UUID. Invalid input returns an error // instead of a zero-valued UUID — silently dropping bad input has caused // data-loss bugs (e.g. DELETE matching no rows, returning 204 success). diff --git a/server/migrations/203_domain_event.down.sql b/server/migrations/203_domain_event.down.sql new file mode 100644 index 00000000000..b31c9668aec --- /dev/null +++ b/server/migrations/203_domain_event.down.sql @@ -0,0 +1,5 @@ +-- Reverses 203_domain_event.up.sql. Dropping the table also drops the +-- sequence it OWNS; the explicit DROP SEQUENCE is a defensive no-op in case +-- the ownership link is ever severed. +DROP TABLE IF EXISTS domain_event; +DROP SEQUENCE IF EXISTS domain_event_seq; diff --git a/server/migrations/203_domain_event.up.sql b/server/migrations/203_domain_event.up.sql new file mode 100644 index 00000000000..ce6b3aa5181 --- /dev/null +++ b/server/migrations/203_domain_event.up.sql @@ -0,0 +1,78 @@ +-- Event Hooks MVP — PR1: the transactional-outbox domain event log (MUL-4332 §4.1). +-- +-- `domain_event` is the persisted source of truth for the future hooks engine. +-- Every domain command that commits a fact (issue created / status changed / +-- assigned, comment created, task completed / failed) will ALSO insert one row +-- here IN THE SAME TRANSACTION, so the fact and its event commit atomically — +-- the classic transactional outbox. A crash after the domain write but before +-- any downstream reaction can therefore never lose the event. +-- +-- PR1 ships the table + the write-path convergence only. There is NO consumer +-- yet: rows land with dispatch_status='pending' and nothing reads them, so this +-- is a zero-behavior-change, additive migration. The matcher/executor that +-- claims pending rows via the lease columns arrives in PR3. +-- +-- The in-memory `events.Bus` (internal/events) is SEPARATE and unchanged — it +-- keeps serving realtime UI fanout. This table is the durable, transactional +-- fact log; the Bus is best-effort post-commit push. They do not replace each +-- other. +-- +-- Workspace DB rules (CLAUDE.md + MUL-4332 §4): NO foreign key, NO cascade — +-- every UUID association is validated in the application layer. Secondary and +-- unique indexes are added in their own single-statement CONCURRENTLY +-- migrations (204–208), never inline, so index builds never take an ACCESS +-- EXCLUSIVE lock during deploy. + +-- Monotonic dispatch / drain boundary. `seq` orders events for stable scanning +-- and the future issue-scope drain boundary; it is NOT a promise of global +-- business ordering across all actions (MUL-4332 §4.1). A sequence (not a +-- serial/identity column) so the ownership is explicit and the down migration +-- can drop it cleanly. +CREATE SEQUENCE IF NOT EXISTS domain_event_seq; + +CREATE TABLE IF NOT EXISTS domain_event ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + seq BIGINT NOT NULL DEFAULT nextval('domain_event_seq'), + + -- Fact envelope. + workspace_id UUID NOT NULL, + type TEXT NOT NULL, -- e.g. issue.status_changed + schema_version INT NOT NULL, -- per-type payload schema version + subject_type TEXT NOT NULL, -- issue | comment | task + subject_id UUID NOT NULL, + actor_type TEXT NOT NULL, -- member | agent | system | hook + actor_id UUID, -- NULL for system actors + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- Causal chain. A human root event has correlation_id = id and hop_count = 0; + -- events produced by a future hook action inherit the correlation, record the + -- producing execution/action, and increment hop_count (guardrail in PR3). + correlation_id UUID NOT NULL, + causation_execution_id UUID, + causation_action_index INT, + hop_count INT NOT NULL DEFAULT 0, + + -- Single-consumer outbox lease. Unused in PR1 (no matcher); rows stay + -- 'pending'. PR3's matcher claims via lease_token/lease_expires_at and + -- advances dispatch_status. + dispatch_status TEXT NOT NULL DEFAULT 'pending' + CHECK (dispatch_status IN ('pending', 'dispatching', 'dispatched', 'failed')), + attempts INT NOT NULL DEFAULT 0, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + dispatched_at TIMESTAMPTZ, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Tie the sequence lifecycle to the column so a table drop reclaims it. +ALTER SEQUENCE domain_event_seq OWNED BY domain_event.seq; + +-- Application-layer integrity only: proactively drop any *_fkey a tool might +-- add, matching the workspace no-FK / no-cascade rule (see 186_autopilot_rule_version). +ALTER TABLE domain_event + DROP CONSTRAINT IF EXISTS domain_event_workspace_id_fkey; + +COMMENT ON TABLE domain_event IS + 'Transactional-outbox domain event log (MUL-4332 §4.1). One row per committed domain fact, written in the same tx as the fact. Source of truth for the hooks engine; no FK, no cascade, app-layer integrity. dispatch_* columns are the single-consumer outbox lease consumed by the PR3 matcher.'; diff --git a/server/migrations/204_domain_event_seq_unique_index.down.sql b/server/migrations/204_domain_event_seq_unique_index.down.sql new file mode 100644 index 00000000000..9bf30a64858 --- /dev/null +++ b/server/migrations/204_domain_event_seq_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_seq; diff --git a/server/migrations/204_domain_event_seq_unique_index.up.sql b/server/migrations/204_domain_event_seq_unique_index.up.sql new file mode 100644 index 00000000000..dc145cc70aa --- /dev/null +++ b/server/migrations/204_domain_event_seq_unique_index.up.sql @@ -0,0 +1,8 @@ +-- Single-statement CONCURRENTLY migration: Postgres rejects CREATE INDEX +-- CONCURRENTLY inside a transaction or a multi-command string, and the +-- migration runner execs each file as one command (see 080/135). +-- +-- Enforces the monotonic-uniqueness invariant of domain_event.seq and serves +-- the ordered `... ORDER BY seq` drain/scan the PR3 matcher will use. +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_seq + ON domain_event (seq); diff --git a/server/migrations/205_domain_event_dispatch_index.down.sql b/server/migrations/205_domain_event_dispatch_index.down.sql new file mode 100644 index 00000000000..80aa4ff5945 --- /dev/null +++ b/server/migrations/205_domain_event_dispatch_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_dispatch; diff --git a/server/migrations/205_domain_event_dispatch_index.up.sql b/server/migrations/205_domain_event_dispatch_index.up.sql new file mode 100644 index 00000000000..b22528011f2 --- /dev/null +++ b/server/migrations/205_domain_event_dispatch_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 204). Backs the PR3 matcher's +-- claim scan for undispatched, now-available events in seq order: +-- WHERE dispatch_status = 'pending' AND available_at <= now() ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_dispatch + ON domain_event (dispatch_status, available_at, seq); diff --git a/server/migrations/206_domain_event_correlation_index.down.sql b/server/migrations/206_domain_event_correlation_index.down.sql new file mode 100644 index 00000000000..3bf1d98cdc5 --- /dev/null +++ b/server/migrations/206_domain_event_correlation_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_correlation; diff --git a/server/migrations/206_domain_event_correlation_index.up.sql b/server/migrations/206_domain_event_correlation_index.up.sql new file mode 100644 index 00000000000..b3f6c0705a6 --- /dev/null +++ b/server/migrations/206_domain_event_correlation_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 204). Backs correlation-chain +-- reads (GET /api/events?correlation_id=) and loop/depth guardrail lookups: +-- WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_correlation + ON domain_event (workspace_id, correlation_id, seq); diff --git a/server/migrations/207_domain_event_type_index.down.sql b/server/migrations/207_domain_event_type_index.down.sql new file mode 100644 index 00000000000..08ae0ccb9b0 --- /dev/null +++ b/server/migrations/207_domain_event_type_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_type; diff --git a/server/migrations/207_domain_event_type_index.up.sql b/server/migrations/207_domain_event_type_index.up.sql new file mode 100644 index 00000000000..56a57e3e025 --- /dev/null +++ b/server/migrations/207_domain_event_type_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 204). Backs per-workspace, +-- per-type event scans used by explain/debug tooling: +-- WHERE workspace_id = $1 AND type = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_type + ON domain_event (workspace_id, type, seq); diff --git a/server/migrations/208_domain_event_subject_index.down.sql b/server/migrations/208_domain_event_subject_index.down.sql new file mode 100644 index 00000000000..6a2a5d1ef06 --- /dev/null +++ b/server/migrations/208_domain_event_subject_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_subject; diff --git a/server/migrations/208_domain_event_subject_index.up.sql b/server/migrations/208_domain_event_subject_index.up.sql new file mode 100644 index 00000000000..143834aaec7 --- /dev/null +++ b/server/migrations/208_domain_event_subject_index.up.sql @@ -0,0 +1,6 @@ +-- Single-statement CONCURRENTLY migration (see 204). Backs "all events about +-- this subject" scans (a given issue / comment / task) for debug + the future +-- stage sensor: +-- WHERE subject_type = $1 AND subject_id = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_subject + ON domain_event (subject_type, subject_id, seq); diff --git a/server/migrations/209_hook.down.sql b/server/migrations/209_hook.down.sql new file mode 100644 index 00000000000..d89376e88b4 --- /dev/null +++ b/server/migrations/209_hook.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook; diff --git a/server/migrations/209_hook.up.sql b/server/migrations/209_hook.up.sql new file mode 100644 index 00000000000..685a669af10 --- /dev/null +++ b/server/migrations/209_hook.up.sql @@ -0,0 +1,24 @@ +-- Event Hooks MVP — stable hook identity and lifecycle ownership. +-- UUID associations are application-validated; do not add FKs or cascades. +CREATE TABLE hook ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL, + name TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + active_revision_id UUID NOT NULL, + scope_type TEXT NOT NULL CHECK (scope_type IN ('workspace', 'issue')), + scope_id UUID, + retire_after_event_seq BIGINT, + origin TEXT NOT NULL DEFAULT 'user' CHECK (origin IN ('user', 'system')), + system_key TEXT, + system_version INT, + creator_actor_type TEXT NOT NULL, + creator_actor_id UUID, + authorization_principal_user_id UUID, + disabled_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + archived_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook IS 'Event Hooks stable identity and lifecycle owner. No foreign keys; application validates all associations.'; diff --git a/server/migrations/210_hook_revision.down.sql b/server/migrations/210_hook_revision.down.sql new file mode 100644 index 00000000000..d023801b896 --- /dev/null +++ b/server/migrations/210_hook_revision.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_revision; diff --git a/server/migrations/210_hook_revision.up.sql b/server/migrations/210_hook_revision.up.sql new file mode 100644 index 00000000000..3123dec7bb6 --- /dev/null +++ b/server/migrations/210_hook_revision.up.sql @@ -0,0 +1,16 @@ +-- Immutable configurations. Updating a hook creates a new revision row. +CREATE TABLE hook_revision ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + hook_id UUID NOT NULL, + revision INT NOT NULL, + event_type TEXT NOT NULL, + match JSONB NOT NULL DEFAULT '{}'::jsonb, + conditions JSONB, + fire_mode TEXT NOT NULL CHECK (fire_mode IN ('per_event', 'rising_edge')), + actions JSONB NOT NULL, + created_by_type TEXT NOT NULL, + created_by_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +COMMENT ON TABLE hook_revision IS 'Immutable Event Hooks configuration revisions. No foreign keys; application validates hook ownership.'; diff --git a/server/migrations/211_automation_state.down.sql b/server/migrations/211_automation_state.down.sql new file mode 100644 index 00000000000..85174bd27eb --- /dev/null +++ b/server/migrations/211_automation_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS automation_state; diff --git a/server/migrations/211_automation_state.up.sql b/server/migrations/211_automation_state.up.sql new file mode 100644 index 00000000000..e688b127285 --- /dev/null +++ b/server/migrations/211_automation_state.up.sql @@ -0,0 +1,12 @@ +-- Small durable, row-lockable state for edge latches, stage frontier and rate buckets. +CREATE TABLE automation_state ( + workspace_id UUID NOT NULL, + state_kind TEXT NOT NULL, + state_key TEXT NOT NULL, + state JSONB NOT NULL, + version BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, state_kind, state_key) +); + +COMMENT ON TABLE automation_state IS 'Durable row-lockable automation state, not an audit log. No foreign keys.'; diff --git a/server/migrations/212_hook_execution.down.sql b/server/migrations/212_hook_execution.down.sql new file mode 100644 index 00000000000..9ca147a0300 --- /dev/null +++ b/server/migrations/212_hook_execution.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_execution; diff --git a/server/migrations/212_hook_execution.up.sql b/server/migrations/212_hook_execution.up.sql new file mode 100644 index 00000000000..b2716853bb7 --- /dev/null +++ b/server/migrations/212_hook_execution.up.sql @@ -0,0 +1,25 @@ +-- Matcher decision and executor trace. The revision is pinned at creation. +CREATE TABLE hook_execution ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL, + hook_id UUID NOT NULL, + hook_revision_id UUID NOT NULL, + event_id UUID NOT NULL, + correlation_id UUID NOT NULL, + status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed', 'skipped')), + skip_reason TEXT, + match_snapshot JSONB, + condition_snapshot JSONB, + current_action_index INT NOT NULL DEFAULT 0, + attempts INT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ, + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + error_code TEXT, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook_execution IS 'Durable matcher/executor trace with revision pinning. No foreign keys; application validates all associations.'; diff --git a/server/migrations/213_hook_action_effect.down.sql b/server/migrations/213_hook_action_effect.down.sql new file mode 100644 index 00000000000..73ceedbcb09 --- /dev/null +++ b/server/migrations/213_hook_action_effect.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_action_effect; diff --git a/server/migrations/213_hook_action_effect.up.sql b/server/migrations/213_hook_action_effect.up.sql new file mode 100644 index 00000000000..2d3a06ed0ee --- /dev/null +++ b/server/migrations/213_hook_action_effect.up.sql @@ -0,0 +1,19 @@ +-- Crash-safe idempotency anchor for one execution action. +CREATE TABLE hook_action_effect ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + effect_key TEXT NOT NULL, + execution_id UUID NOT NULL, + action_index INT NOT NULL, + action_type TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('running', 'succeeded', 'failed', 'skipped')), + resolved_input JSONB, + output_type TEXT, + output_id UUID, + attempts INT NOT NULL DEFAULT 0, + error_code TEXT, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook_action_effect IS 'One durable idempotency record per hook execution action. No foreign keys.'; diff --git a/server/migrations/214_hook_revision_unique_index.down.sql b/server/migrations/214_hook_revision_unique_index.down.sql new file mode 100644 index 00000000000..147f1b57d6c --- /dev/null +++ b/server/migrations/214_hook_revision_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_revision_unique; diff --git a/server/migrations/214_hook_revision_unique_index.up.sql b/server/migrations/214_hook_revision_unique_index.up.sql new file mode 100644 index 00000000000..c33e02ff018 --- /dev/null +++ b/server/migrations/214_hook_revision_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_revision_unique ON hook_revision (hook_id, revision); diff --git a/server/migrations/215_hook_execution_unique_index.down.sql b/server/migrations/215_hook_execution_unique_index.down.sql new file mode 100644 index 00000000000..7319449246a --- /dev/null +++ b/server/migrations/215_hook_execution_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_execution_hook_event; diff --git a/server/migrations/215_hook_execution_unique_index.up.sql b/server/migrations/215_hook_execution_unique_index.up.sql new file mode 100644 index 00000000000..c9e62d13eb4 --- /dev/null +++ b/server/migrations/215_hook_execution_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_execution_hook_event ON hook_execution (hook_id, event_id); diff --git a/server/migrations/216_hook_effect_key_unique_index.down.sql b/server/migrations/216_hook_effect_key_unique_index.down.sql new file mode 100644 index 00000000000..1494263f79c --- /dev/null +++ b/server/migrations/216_hook_effect_key_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_action_effect_key; diff --git a/server/migrations/216_hook_effect_key_unique_index.up.sql b/server/migrations/216_hook_effect_key_unique_index.up.sql new file mode 100644 index 00000000000..a9a07f65651 --- /dev/null +++ b/server/migrations/216_hook_effect_key_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_action_effect_key ON hook_action_effect (effect_key); diff --git a/server/migrations/217_hook_execution_lease_index.down.sql b/server/migrations/217_hook_execution_lease_index.down.sql new file mode 100644 index 00000000000..a07995618e0 --- /dev/null +++ b/server/migrations/217_hook_execution_lease_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_execution_lease; diff --git a/server/migrations/217_hook_execution_lease_index.up.sql b/server/migrations/217_hook_execution_lease_index.up.sql new file mode 100644 index 00000000000..8025152b492 --- /dev/null +++ b/server/migrations/217_hook_execution_lease_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY idx_hook_execution_lease ON hook_execution (status, next_attempt_at, lease_expires_at, created_at); diff --git a/server/migrations/218_hook_scope_index.down.sql b/server/migrations/218_hook_scope_index.down.sql new file mode 100644 index 00000000000..fc48c705056 --- /dev/null +++ b/server/migrations/218_hook_scope_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_scope_active; diff --git a/server/migrations/218_hook_scope_index.up.sql b/server/migrations/218_hook_scope_index.up.sql new file mode 100644 index 00000000000..a1fed9907d3 --- /dev/null +++ b/server/migrations/218_hook_scope_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY idx_hook_scope_active ON hook (workspace_id, scope_type, scope_id) WHERE archived_at IS NULL; diff --git a/server/pkg/db/generated/agent.sql.go b/server/pkg/db/generated/agent.sql.go index d61bf0c163f..51de3c555b6 100644 --- a/server/pkg/db/generated/agent.sql.go +++ b/server/pkg/db/generated/agent.sql.go @@ -1925,124 +1925,6 @@ func (q *Queries) DeleteSystemAgentByID(ctx context.Context, id pgtype.UUID) err return err } -const expireStaleQueuedTasks = `-- name: ExpireStaleQueuedTasks :many -WITH victims AS ( - SELECT id FROM agent_task_queue - WHERE status = 'queued' - AND created_at < now() - make_interval(secs => $1::double precision) - ORDER BY created_at ASC - LIMIT $2::int - FOR UPDATE SKIP LOCKED -) -UPDATE agent_task_queue t -SET status = 'failed', - completed_at = now(), - error = 'task expired in queue', - failure_reason = 'queued_expired', - prepare_lease_expires_at = NULL -FROM victims v -WHERE t.id = v.id - AND t.status = 'queued' - AND t.created_at < now() - make_interval(secs => $1::double precision) -RETURNING t.id, t.agent_id, t.issue_id, t.status, t.priority, t.dispatched_at, t.started_at, t.completed_at, t.result, t.error, t.created_at, t.context, t.runtime_id, t.session_id, t.work_dir, t.trigger_comment_id, t.chat_session_id, t.autopilot_run_id, t.attempt, t.max_attempts, t.parent_task_id, t.failure_reason, t.trigger_summary, t.force_fresh_session, t.is_leader_task, t.wait_reason, t.initiator_user_id, t.handoff_note, t.prepare_lease_expires_at, t.squad_id, t.runtime_mcp_overlay, t.escalation_for_task_id, t.fire_at, t.originator_user_id, t.runtime_connected_apps, t.coalesced_comment_ids, t.delivered_comment_ids, t.chat_input_task_id, t.chat_finalize_deferred_at, t.originator_source, t.delegated_from_task_id, t.retry_of_task_id, t.rerun_of_task_id, t.rule_version_id, t.trigger_evidence_kind, t.trigger_evidence_ref_id, t.accountable_user_id -` - -type ExpireStaleQueuedTasksParams struct { - TtlSecs float64 `json:"ttl_secs"` - MaxPerTick int32 `json:"max_per_tick"` -} - -// Fails tasks that have been sitting in 'queued' for longer than the TTL. -// This is the cleanup arm of the MUL-1899 "queued backlog" fix: even with the -// new dispatch-time admission gate that refuses to enqueue when the runtime -// is offline, we still need to drain the historical 87k+ doomed rows and -// handle edge cases where a runtime goes offline AFTER a task is already -// queued (the admission check protects new enqueues, not in-flight queue -// depth). -// -// Concurrency safety: the daemon's claim path may race with this sweeper to -// transition the same row out of 'queued'. We protect against that two -// ways: -// 1. The CTE selects victims with FOR UPDATE SKIP LOCKED so a row that is -// currently being claimed (or otherwise locked) is skipped — no lock -// contention with the dispatch path, and we won't queue up behind it. -// 2. The outer UPDATE re-checks status='queued' AND the TTL predicate at -// apply time. If a daemon claimed the row between selection and update -// (e.g. lock released after the claim transaction commits), the row is -// already 'dispatched'/'running' and the WHERE clause filters it out -// so we cannot clobber an in-flight task. -// -// Capped via LIMIT inside the CTE so a single sweep tick cannot monopolise -// the DB when the backlog is large — the sweeper drains the rest on -// subsequent ticks. -func (q *Queries) ExpireStaleQueuedTasks(ctx context.Context, arg ExpireStaleQueuedTasksParams) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, expireStaleQueuedTasks, arg.TtlSecs, arg.MaxPerTick) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const extendAgentTaskPrepareLease = `-- name: ExtendAgentTaskPrepareLease :one UPDATE agent_task_queue SET prepare_lease_expires_at = now() + make_interval(secs => $3::double precision) @@ -2209,74 +2091,36 @@ func (q *Queries) FailAgentTask(ctx context.Context, arg FailAgentTaskParams) (A return i, err } -const failStaleTasks = `-- name: FailStaleTasks :many +const failAgentTasksByIDs = `-- name: FailAgentTasksByIDs :many UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'task timed out', - failure_reason = 'timeout', +SET status = 'failed', + completed_at = now(), + error = $1, + failure_reason = $2, + wait_reason = NULL, prepare_lease_expires_at = NULL -WHERE ( - status = 'dispatched' - AND dispatched_at < now() - make_interval(secs => $1::double precision) - AND (prepare_lease_expires_at IS NULL OR prepare_lease_expires_at < now()) - ) - OR ( - status = 'running' - AND started_at < now() - make_interval(secs => $2::double precision) - AND NOT EXISTS ( - SELECT 1 FROM agent_runtime r - WHERE r.id = agent_task_queue.runtime_id - AND r.status = 'online' - AND r.last_seen_at >= now() - make_interval(secs => $3::double precision) - ) - ) +WHERE id = ANY($3::uuid[]) + AND status IN ('queued', 'dispatched', 'running', 'waiting_local_directory') RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id ` -type FailStaleTasksParams struct { - DispatchTimeoutSecs float64 `json:"dispatch_timeout_secs"` - RunningTimeoutSecs float64 `json:"running_timeout_secs"` - RuntimeStaleSecs float64 `json:"runtime_stale_secs"` +type FailAgentTasksByIDsParams struct { + Error pgtype.Text `json:"error"` + FailureReason pgtype.Text `json:"failure_reason"` + Ids []pgtype.UUID `json:"ids"` } -// Fails tasks stuck in dispatched/running beyond the given thresholds. -// -// Each branch pairs a wall-clock deadline with a task-appropriate liveness -// signal, so the sweeper only kills tasks whose owning daemon is no longer -// proving it is alive: -// -// - Dispatched: `prepare_lease_expires_at` is refreshed every 15s by the -// daemon between claim and StartTask (see startTaskPrepareLeaseExtender). -// A live lease excludes the row. -// -// - Running: no per-task lease is renewed once StartTask fires, so we key -// off the daemon-wide heartbeat instead — `agent_runtime.last_seen_at`, -// which the daemon bumps every ~15s while it is up. A running task whose -// runtime is `online` AND whose `last_seen_at` is within -// @runtime_stale_secs is treated as alive and is NOT killed by this -// wall-clock backstop, even after `started_at` exceeds the running -// timeout. This is what lets healthy multi-hour research / training runs -// survive on self-hosted deployments (MUL-4107): the daemon side is -// bounded only by inactivity watchdogs (idle / per-tool), so the -// server-side wall clock must not shadow that with a coarser cap. -// -// The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` -// (which mixes DB `last_seen_at` with the Redis LivenessStore and calls -// `FailTasksForOfflineRuntimes` in the same tick). The wall-clock branch -// here is a defensive backstop for pathological cases where a runtime row -// somehow retains status='online' with a stale DB heartbeat for longer than -// the wall clock allows. -// -// runtime_id IS NULL: a running row with no runtime is by definition not -// proving liveness, so the wall clock is allowed to fire — same shape as -// the legacy pure-wall-clock behavior for that (rare / historical) case. -// -// waiting_local_directory rows are intentionally excluded: the daemon owns -// the wait (with its own ctx-driven timeout) and a legitimate queue ahead -// of this task can exceed the dispatch / running timeouts without being -// "stuck". If the daemon dies, RecoverOrphanedTasksForRuntime reclaims -// those rows at restart. -func (q *Queries) FailStaleTasks(ctx context.Context, arg FailStaleTasksParams) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, failStaleTasks, arg.DispatchTimeoutSecs, arg.RunningTimeoutSecs, arg.RuntimeStaleSecs) +// Fails a specific, already-resolved set of tasks by id — the terminal half of +// the MUL-4332 poison-isolated bulk fail (review point 2). The caller has just +// selected these ids with SelectStaleTasksToFail / SelectExpiredQueuedTasks / +// SelectTasksForOfflineRuntimes / SelectOrphanedTasksForRuntime FOR UPDATE in +// the SAME transaction, so the rows are locked and cannot have transitioned; +// the status guard is a defensive backstop that keeps this idempotent if the id +// set is ever reused. @error / @failure_reason are supplied per sweeper. Both +// wait_reason and prepare_lease_expires_at are cleared (clearing a NULL is a +// no-op) so this one query serves every bulk-fail caller. +func (q *Queries) FailAgentTasksByIDs(ctx context.Context, arg FailAgentTasksByIDsParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, failAgentTasksByIDs, arg.Error, arg.FailureReason, arg.Ids) if err != nil { return nil, err } @@ -4455,92 +4299,6 @@ func (q *Queries) ReclaimStaleDispatchedTasksForRuntimes(ctx context.Context, ar return items, nil } -const recoverOrphanedTasksForRuntime = `-- name: RecoverOrphanedTasksForRuntime :many -UPDATE agent_task_queue -SET status = 'failed', - completed_at = now(), - error = 'daemon restarted while task was in flight', - failure_reason = 'runtime_recovery', - wait_reason = NULL, - prepare_lease_expires_at = NULL -WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') -RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id -` - -// Called by the daemon at startup. Atomically fails any dispatched/running/ -// waiting_local_directory task that the prior incarnation of this runtime -// owned but did not finalize. Returns the failed rows so callers can hand -// them to the auto-retry path. waiting_local_directory rows are included -// because the daemon holding the path lock is the same process that just -// died — without us, the row would sit waiting forever. -func (q *Queries) RecoverOrphanedTasksForRuntime(ctx context.Context, runtimeID pgtype.UUID) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, recoverOrphanedTasksForRuntime, runtimeID) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const refreshAgentStatusFromTasks = `-- name: RefreshAgentStatusFromTasks :one UPDATE agent AS a SET status = CASE WHEN EXISTS ( @@ -4706,6 +4464,374 @@ func (q *Queries) RestoreAgent(ctx context.Context, id pgtype.UUID) (Agent, erro return i, err } +const selectExpiredQueuedTasks = `-- name: SelectExpiredQueuedTasks :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE status = 'queued' + AND created_at < now() - make_interval(secs => $1::double precision) +ORDER BY created_at ASC +LIMIT $2::int +FOR UPDATE SKIP LOCKED +` + +type SelectExpiredQueuedTasksParams struct { + TtlSecs float64 `json:"ttl_secs"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Selects (and row-locks) up to @max_per_tick tasks sitting in 'queued' past the +// TTL — the cleanup arm of the MUL-1899 "queued backlog" fix (drains the +// historical 87k+ doomed rows and catches the case where a runtime goes offline +// AFTER a task is already queued). +// +// Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +// caller resolves each task's workspace, fails only the resolvable set via +// FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +// FOR UPDATE SKIP LOCKED skips rows a daemon is currently claiming, so we never +// contend with the dispatch path or clobber an in-flight task; because the fail +// runs in this same transaction the locked rows cannot transition out of 'queued' +// underneath us. The LIMIT bounds the lock hold; the rest drain on later ticks. +func (q *Queries) SelectExpiredQueuedTasks(ctx context.Context, arg SelectExpiredQueuedTasksParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectExpiredQueuedTasks, arg.TtlSecs, arg.MaxPerTick) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectOrphanedTasksForRuntime = `-- name: SelectOrphanedTasksForRuntime :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') + AND ( + $2::timestamptz IS NULL + OR created_at > $2::timestamptz + OR (created_at = $2::timestamptz AND id > $3::uuid) + ) +ORDER BY created_at ASC, id ASC +LIMIT $4::int +FOR UPDATE +` + +type SelectOrphanedTasksForRuntimeParams struct { + RuntimeID pgtype.UUID `json:"runtime_id"` + AfterCreatedAt pgtype.Timestamptz `json:"after_created_at"` + AfterID pgtype.UUID `json:"after_id"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Called by the daemon at startup. Selects (and row-locks) up to @max_per_tick +// dispatched/running/waiting_local_directory tasks that the prior incarnation of +// this runtime owned but did not finalize. waiting_local_directory rows are +// included because the daemon holding the path lock is the same process that +// just died — without us, the row would sit waiting forever. +// +// Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +// caller resolves each task's workspace, fails only the resolvable set via +// FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +// The LIMIT bounds the lock hold. +// +// Plain FOR UPDATE (NOT SKIP LOCKED — review round 4, point 1). Unlike the +// sweepers, which re-scan from the start every tick so a skipped row is simply +// retried next tick, this query pages forward with a keyset cursor that advances +// permanently past whatever a page returned. If SKIP LOCKED silently dropped an +// older orphan that happened to be briefly locked (a sweep, a stale-dispatch +// reclaim), the cursor — filled by the newer rows behind it — would step past that +// older row and NO later page could ever select it again; the runtime is already +// back `online`, so the offline sweep won't reap it either, and it leaks forever. +// Plain FOR UPDATE instead WAITS for the lock and, once it releases, re-checks the +// row against the WHERE (Postgres EvalPlanQual): still dispatched/running → +// included on this page; already failed by whoever held the lock → correctly +// excluded. The bounded page size keeps that wait short, and recovery only races +// short sweeper/reclaim transactions (both SKIP LOCKED, so they never wait — no +// deadlock cycle is possible). +// +// Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts +// the runtime back to `online`, so the every-tick offline sweep will NOT reap an +// orphan the daemon leaves past this page — the recovery must therefore drain +// itself. Callers page by (created_at, id) via @after_created_at / @after_id (NULL +// on the first page) and repeat until a short page. Because the cursor advances +// over EVERY returned candidate — including a poison (unresolvable-workspace) row +// the caller skips rather than fails — a page full of poison at the front can no +// longer pin the drain in place: the next page steps past it to the healthy rows +// behind it. (A plain re-select of the oldest rows would loop on the poison +// forever.) Ordering matches the keyset so pages are stable and non-overlapping. +func (q *Queries) SelectOrphanedTasksForRuntime(ctx context.Context, arg SelectOrphanedTasksForRuntimeParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectOrphanedTasksForRuntime, + arg.RuntimeID, + arg.AfterCreatedAt, + arg.AfterID, + arg.MaxPerTick, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectStaleTasksToFail = `-- name: SelectStaleTasksToFail :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE ( + status = 'dispatched' + AND dispatched_at < now() - make_interval(secs => $1::double precision) + AND (prepare_lease_expires_at IS NULL OR prepare_lease_expires_at < now()) + ) + OR ( + status = 'running' + AND started_at < now() - make_interval(secs => $2::double precision) + AND NOT EXISTS ( + SELECT 1 FROM agent_runtime r + WHERE r.id = agent_task_queue.runtime_id + AND r.status = 'online' + AND r.last_seen_at >= now() - make_interval(secs => $3::double precision) + ) + ) +ORDER BY started_at ASC NULLS FIRST, dispatched_at ASC NULLS FIRST +LIMIT $4::int +FOR UPDATE SKIP LOCKED +` + +type SelectStaleTasksToFailParams struct { + DispatchTimeoutSecs float64 `json:"dispatch_timeout_secs"` + RunningTimeoutSecs float64 `json:"running_timeout_secs"` + RuntimeStaleSecs float64 `json:"runtime_stale_secs"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Selects (and row-locks) up to @max_per_tick tasks stuck in dispatched/running +// beyond the given thresholds. Candidate half of the MUL-4332 poison-isolated +// fail path (review point 2): the caller resolves each task's workspace, fails +// only the resolvable set via FailAgentTasksByIDs and emits its task.failed event +// in the SAME transaction. FOR UPDATE SKIP LOCKED keeps the backstop off rows a +// daemon is actively claiming; the LIMIT bounds the lock hold. +// +// Each branch pairs a wall-clock deadline with a task-appropriate liveness +// signal, so the sweeper only kills tasks whose owning daemon is no longer +// proving it is alive: +// +// - Dispatched: `prepare_lease_expires_at` is refreshed every 15s by the +// daemon between claim and StartTask (see startTaskPrepareLeaseExtender). +// A live lease excludes the row. +// +// - Running: no per-task lease is renewed once StartTask fires, so we key +// off the daemon-wide heartbeat instead — `agent_runtime.last_seen_at`, +// which the daemon bumps every ~15s while it is up. A running task whose +// runtime is `online` AND whose `last_seen_at` is within +// @runtime_stale_secs is treated as alive and is NOT killed by this +// wall-clock backstop, even after `started_at` exceeds the running +// timeout. This is what lets healthy multi-hour research / training runs +// survive on self-hosted deployments (MUL-4107): the daemon side is +// bounded only by inactivity watchdogs (idle / per-tool), so the +// server-side wall clock must not shadow that with a coarser cap. +// +// The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` +// (which mixes DB `last_seen_at` with the Redis LivenessStore and fails +// offline-runtime tasks via SelectTasksForOfflineRuntimes in the same tick). +// The wall-clock branch +// here is a defensive backstop for pathological cases where a runtime row +// somehow retains status='online' with a stale DB heartbeat for longer than +// the wall clock allows. +// +// runtime_id IS NULL: a running row with no runtime is by definition not +// proving liveness, so the wall clock is allowed to fire — same shape as +// the legacy pure-wall-clock behavior for that (rare / historical) case. +// +// waiting_local_directory rows are intentionally excluded: the daemon owns +// the wait (with its own ctx-driven timeout) and a legitimate queue ahead +// of this task can exceed the dispatch / running timeouts without being +// "stuck". If the daemon dies, SelectOrphanedTasksForRuntime reclaims +// those rows at restart. +func (q *Queries) SelectStaleTasksToFail(ctx context.Context, arg SelectStaleTasksToFailParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectStaleTasksToFail, + arg.DispatchTimeoutSecs, + arg.RunningTimeoutSecs, + arg.RuntimeStaleSecs, + arg.MaxPerTick, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const setTaskDeliveredCommentIDs = `-- name: SetTaskDeliveredCommentIDs :one UPDATE agent_task_queue SET delivered_comment_ids = $1::uuid[] diff --git a/server/pkg/db/generated/automation_state.sql.go b/server/pkg/db/generated/automation_state.sql.go new file mode 100644 index 00000000000..73402ff0810 --- /dev/null +++ b/server/pkg/db/generated/automation_state.sql.go @@ -0,0 +1,77 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: automation_state.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getAutomationStateForUpdate = `-- name: GetAutomationStateForUpdate :one + +SELECT workspace_id, state_kind, state_key, state, version, updated_at FROM automation_state +WHERE workspace_id = $1 AND state_kind = $2 AND state_key = $3 +FOR UPDATE +` + +type GetAutomationStateForUpdateParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` +} + +// Durable row-lockable automation state (MUL-4332): rising-edge latches (and, +// later, stage frontier + rate buckets). Not an audit log; one row per key. +// Row-lock a state key so concurrent matchers serialize their read-modify-write +// of the same latch. Returns no rows the first time a key is used. +func (q *Queries) GetAutomationStateForUpdate(ctx context.Context, arg GetAutomationStateForUpdateParams) (AutomationState, error) { + row := q.db.QueryRow(ctx, getAutomationStateForUpdate, arg.WorkspaceID, arg.StateKind, arg.StateKey) + var i AutomationState + err := row.Scan( + &i.WorkspaceID, + &i.StateKind, + &i.StateKey, + &i.State, + &i.Version, + &i.UpdatedAt, + ) + return i, err +} + +const upsertAutomationState = `-- name: UpsertAutomationState :one +INSERT INTO automation_state (workspace_id, state_kind, state_key, state, version) +VALUES ($1, $2, $3, $4, 0) +ON CONFLICT (workspace_id, state_kind, state_key) +DO UPDATE SET state = EXCLUDED.state, version = automation_state.version + 1, updated_at = now() +RETURNING workspace_id, state_kind, state_key, state, version, updated_at +` + +type UpsertAutomationStateParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` + State []byte `json:"state"` +} + +func (q *Queries) UpsertAutomationState(ctx context.Context, arg UpsertAutomationStateParams) (AutomationState, error) { + row := q.db.QueryRow(ctx, upsertAutomationState, + arg.WorkspaceID, + arg.StateKind, + arg.StateKey, + arg.State, + ) + var i AutomationState + err := row.Scan( + &i.WorkspaceID, + &i.StateKind, + &i.StateKey, + &i.State, + &i.Version, + &i.UpdatedAt, + ) + return i, err +} diff --git a/server/pkg/db/generated/domain_event.sql.go b/server/pkg/db/generated/domain_event.sql.go new file mode 100644 index 00000000000..55d712a916d --- /dev/null +++ b/server/pkg/db/generated/domain_event.sql.go @@ -0,0 +1,484 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: domain_event.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const claimOneEventWithCandidates = `-- name: ClaimOneEventWithCandidates :many + +WITH claimed AS ( + UPDATE domain_event + SET dispatch_status = 'dispatching', + lease_token = $1, + -- Derive the deadline from the DATABASE clock, the same clock the ownership + -- predicate compares it against. Computing it application-side would let + -- app/DB clock skew grant a lease that is already expired, or one that + -- outlives its intended TTL. + lease_expires_at = clock_timestamp() + make_interval(secs => $2::float8), + attempts = attempts + 1 + WHERE id = ( + SELECT id FROM domain_event + WHERE available_at <= now() + AND (dispatch_status = 'pending' + OR (dispatch_status = 'dispatching' AND lease_expires_at < now())) + ORDER BY seq ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, workspace_id, type, subject_id, actor_type, actor_id, + payload, correlation_id, hop_count +) +SELECT + c.id AS event_id, + c.workspace_id AS event_workspace_id, + c.type AS event_type, + c.subject_id AS event_subject_id, + c.actor_type AS event_actor_type, + c.actor_id AS event_actor_id, + c.payload AS event_payload, + c.correlation_id AS event_correlation_id, + c.hop_count AS event_hop_count, + cand.hook_id AS hook_id, + cand.revision_id AS revision_id, + cand.match AS match, + cand.conditions AS conditions, + -- COALESCE so the no-candidate row (every cand.* NULL) still scans into a + -- non-nullable string; callers gate on hook_id being present. + COALESCE(cand.fire_mode, '')::text AS fire_mode +FROM claimed c +LEFT JOIN LATERAL ( + SELECT h.id AS hook_id, h.created_at AS hook_created_at, + r.id AS revision_id, r.match, r.conditions, r.fire_mode + FROM hook h + JOIN hook_revision r ON r.id = h.active_revision_id + WHERE h.workspace_id = c.workspace_id + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = c.type +) cand ON true +ORDER BY cand.hook_created_at ASC NULLS LAST, cand.hook_id ASC +` + +type ClaimOneEventWithCandidatesParams struct { + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseTtlSeconds float64 `json:"lease_ttl_seconds"` +} + +type ClaimOneEventWithCandidatesRow struct { + EventID pgtype.UUID `json:"event_id"` + EventWorkspaceID pgtype.UUID `json:"event_workspace_id"` + EventType string `json:"event_type"` + EventSubjectID pgtype.UUID `json:"event_subject_id"` + EventActorType string `json:"event_actor_type"` + EventActorID pgtype.UUID `json:"event_actor_id"` + EventPayload []byte `json:"event_payload"` + EventCorrelationID pgtype.UUID `json:"event_correlation_id"` + EventHopCount int32 `json:"event_hop_count"` + HookID pgtype.UUID `json:"hook_id"` + RevisionID pgtype.UUID `json:"revision_id"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` +} + +// NOTE: the retention/TTL delete is intentionally NOT defined in PR1. The +// correct predicate is "dispatched AND older than TTL AND every related +// hook_execution is terminal" (MUL-4332 §4.1/§9), and hook_execution does not +// exist until PR3. Shipping a weaker "dispatched + TTL" delete now would risk +// reclaiming still-executing audit sources the moment PR3 enables dispatching +// (review point 5). The query lands in PR3 with the full terminal predicate. +// The durable matcher's claim AND revision pin, in a SINGLE statement (MUL-4332 PR3 +// review round: matcher point 1). It leases exactly one undispatched, now-available +// event in seq order and, in the same statement, materializes that event's complete +// (hook_id, revision_id) candidate set with each revision's configuration. +// +// One statement is what makes the pin real. PostgreSQL's default READ COMMITTED +// gives every statement its own snapshot, so claiming in one statement and then +// selecting candidates in another — even inside one transaction — lets a revision +// edit committed in between change this event's decision, and lets two candidates be +// decided against revisions from different instants. Here the claim and the whole +// candidate set share one snapshot, so the pinned revisions are exactly those active +// at claim time (§5.1 "使用 matcher claim 时的当前 enabled revision"). +// +// The LEFT JOIN LATERAL means an event with no candidates still returns exactly one +// row (hook_id NULL), so the caller can distinguish "claimed, nothing matched" from +// "nothing to claim" (zero rows). Reclaims events left 'dispatching' by a crashed +// matcher once their lease expires, so processing is at-least-once. FOR UPDATE SKIP +// LOCKED lets multiple matchers share the queue and keeps the claimed row locked for +// the rest of the transaction. Rides idx_domain_event_dispatch. +// +// Issue scope is lifecycle ownership only and does NOT restrict the event subject — +// that is the job of `when` — so scope is not a filter here. +func (q *Queries) ClaimOneEventWithCandidates(ctx context.Context, arg ClaimOneEventWithCandidatesParams) ([]ClaimOneEventWithCandidatesRow, error) { + rows, err := q.db.Query(ctx, claimOneEventWithCandidates, arg.LeaseToken, arg.LeaseTtlSeconds) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ClaimOneEventWithCandidatesRow{} + for rows.Next() { + var i ClaimOneEventWithCandidatesRow + if err := rows.Scan( + &i.EventID, + &i.EventWorkspaceID, + &i.EventType, + &i.EventSubjectID, + &i.EventActorType, + &i.EventActorID, + &i.EventPayload, + &i.EventCorrelationID, + &i.EventHopCount, + &i.HookID, + &i.RevisionID, + &i.Match, + &i.Conditions, + &i.FireMode, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const countDomainEventsBySubject = `-- name: CountDomainEventsBySubject :one +SELECT count(*) FROM domain_event +WHERE subject_type = $1 + AND subject_id = $2 +` + +type CountDomainEventsBySubjectParams struct { + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` +} + +func (q *Queries) CountDomainEventsBySubject(ctx context.Context, arg CountDomainEventsBySubjectParams) (int64, error) { + row := q.db.QueryRow(ctx, countDomainEventsBySubject, arg.SubjectType, arg.SubjectID) + var count int64 + err := row.Scan(&count) + return count, err +} + +const createDomainEvent = `-- name: CreateDomainEvent :one + +INSERT INTO domain_event ( + id, + workspace_id, + type, + schema_version, + subject_type, + subject_id, + actor_type, + actor_id, + payload, + correlation_id, + causation_execution_id, + causation_action_index, + hop_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 +) +RETURNING id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at +` + +type CreateDomainEventParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID pgtype.UUID `json:"actor_id"` + Payload []byte `json:"payload"` + CorrelationID pgtype.UUID `json:"correlation_id"` + CausationExecutionID pgtype.UUID `json:"causation_execution_id"` + CausationActionIndex pgtype.Int4 `json:"causation_action_index"` + HopCount int32 `json:"hop_count"` +} + +// Transactional-outbox domain event log (MUL-4332 §4.1). CreateDomainEvent is +// the only write; callers invoke it on a *db.Queries already bound to their +// own pgx.Tx (WithTx) so the event commits atomically with the domain fact. +// dispatch_status ('pending'), available_at (now()), attempts (0), seq +// (nextval) and created_at (now()) all come from column defaults — this is a +// root outbox write, never a re-queue, so the writer never sets them. +func (q *Queries) CreateDomainEvent(ctx context.Context, arg CreateDomainEventParams) (DomainEvent, error) { + row := q.db.QueryRow(ctx, createDomainEvent, + arg.ID, + arg.WorkspaceID, + arg.Type, + arg.SchemaVersion, + arg.SubjectType, + arg.SubjectID, + arg.ActorType, + arg.ActorID, + arg.Payload, + arg.CorrelationID, + arg.CausationExecutionID, + arg.CausationActionIndex, + arg.HopCount, + ) + var i DomainEvent + err := row.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ) + return i, err +} + +const deferDomainEventDispatch = `-- name: DeferDomainEventDispatch :execrows +UPDATE domain_event +SET available_at = now() + make_interval(secs => $2::int), + attempts = attempts + 1 +WHERE id = $1 +` + +type DeferDomainEventDispatchParams struct { + ID pgtype.UUID `json:"id"` + BackoffSeconds int32 `json:"backoff_seconds"` +} + +// Back off one event after a transient dispatch failure. The failed decision rolled +// back (including its claim), so without this the event would sit at the head of the +// queue and be re-claimed immediately, spinning on the same failure and starving +// everything behind it. +func (q *Queries) DeferDomainEventDispatch(ctx context.Context, arg DeferDomainEventDispatchParams) (int64, error) { + result, err := q.db.Exec(ctx, deferDomainEventDispatch, arg.ID, arg.BackoffSeconds) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getDomainEvent = `-- name: GetDomainEvent :one +SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event +WHERE id = $1 +` + +func (q *Queries) GetDomainEvent(ctx context.Context, id pgtype.UUID) (DomainEvent, error) { + row := q.db.QueryRow(ctx, getDomainEvent, id) + var i DomainEvent + err := row.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ) + return i, err +} + +const getOwnedDomainEventForDispatch = `-- name: GetOwnedDomainEventForDispatch :one +SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +FOR UPDATE +` + +type GetOwnedDomainEventForDispatchParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +// Row-lock one claimed event and assert lease OWNERSHIP before the matcher writes +// any decision (MUL-4332 PR3 review round: matcher point 2). The predicate is +// deliberately identical to the one MarkDomainEventDispatched/Failed use, and both +// evaluate the expiry against DATABASE clock time (clock_timestamp(), not now(), +// which is frozen at transaction start): a worker holding the right token whose +// lease has nonetheless expired is NOT the owner and must write nothing. Returning +// no rows is the fail-closed signal. +func (q *Queries) GetOwnedDomainEventForDispatch(ctx context.Context, arg GetOwnedDomainEventForDispatchParams) (DomainEvent, error) { + row := q.db.QueryRow(ctx, getOwnedDomainEventForDispatch, arg.ID, arg.LeaseToken) + var i DomainEvent + err := row.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ) + return i, err +} + +const listDomainEventsByCorrelation = `-- name: ListDomainEventsByCorrelation :many +SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event +WHERE workspace_id = $1 + AND correlation_id = $2 +ORDER BY seq ASC +LIMIT $3 +` + +type ListDomainEventsByCorrelationParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + Limit int32 `json:"limit"` +} + +// Bounded correlation-chain read for the debug API. The LIMIT is pushed into the +// query (not applied after loading the whole chain) and rides the +// (workspace_id, correlation_id, seq) index (MUL-4332 PR3 review round: correlation). +func (q *Queries) ListDomainEventsByCorrelation(ctx context.Context, arg ListDomainEventsByCorrelationParams) ([]DomainEvent, error) { + rows, err := q.db.Query(ctx, listDomainEventsByCorrelation, arg.WorkspaceID, arg.CorrelationID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DomainEvent{} + for rows.Next() { + var i DomainEvent + if err := rows.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markDomainEventDispatched = `-- name: MarkDomainEventDispatched :execrows +UPDATE domain_event +SET dispatch_status = 'dispatched', + dispatched_at = now(), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type MarkDomainEventDispatchedParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +// Finalize a matched event under the SAME ownership predicate as the entry +// assertion, so a lease that expired mid-decision cannot commit the decision. The +// caller MUST assert this returns exactly 1 — a 0 means ownership was lost and the +// whole transaction (every execution and latch it wrote) must roll back. +// dispatched_at is set here (and only here) so the retention/audit boundary can +// rely on "dispatched ⇒ dispatched_at". +func (q *Queries) MarkDomainEventDispatched(ctx context.Context, arg MarkDomainEventDispatchedParams) (int64, error) { + result, err := q.db.Exec(ctx, markDomainEventDispatched, arg.ID, arg.LeaseToken) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const markDomainEventFailed = `-- name: MarkDomainEventFailed :execrows +UPDATE domain_event +SET dispatch_status = 'failed', + dispatched_at = now(), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type MarkDomainEventFailedParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +// Terminal failure for an event the matcher can never decode (a malformed payload +// fails identically on every retry). Recording it 'failed' instead of leaving it +// pending stops one poison event from being re-leased forever. Same ownership +// predicate as the dispatched path. +func (q *Queries) MarkDomainEventFailed(ctx context.Context, arg MarkDomainEventFailedParams) (int64, error) { + result, err := q.db.Exec(ctx, markDomainEventFailed, arg.ID, arg.LeaseToken) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/server/pkg/db/generated/hook.sql.go b/server/pkg/db/generated/hook.sql.go new file mode 100644 index 00000000000..6d9d7c5df9a --- /dev/null +++ b/server/pkg/db/generated/hook.sql.go @@ -0,0 +1,1229 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: hook.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const advanceHookExecutionAction = `-- name: AdvanceHookExecutionAction :execrows +UPDATE hook_execution +SET current_action_index = $3 +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type AdvanceHookExecutionActionParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + NextActionIndex int32 `json:"next_action_index"` +} + +// Move the action cursor past a completed action, under the ownership predicate. It +// runs in the SAME transaction as that action's target write and effect, so an action +// can never commit without its cursor advancing. +func (q *Queries) AdvanceHookExecutionAction(ctx context.Context, arg AdvanceHookExecutionActionParams) (int64, error) { + result, err := q.db.Exec(ctx, advanceHookExecutionAction, arg.ID, arg.LeaseToken, arg.NextActionIndex) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const archiveHook = `-- name: ArchiveHook :one +UPDATE hook SET + archived_at = now(), + enabled = false, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type ArchiveHookParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Soft archive (DELETE). Existing revisions / executions / effects are retained +// for audit; the hook is also disabled so nothing can match it. +func (q *Queries) ArchiveHook(ctx context.Context, arg ArchiveHookParams) (Hook, error) { + row := q.db.QueryRow(ctx, archiveHook, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const claimOneHookExecution = `-- name: ClaimOneHookExecution :one +UPDATE hook_execution +SET status = 'running', + lease_token = $1, + lease_expires_at = clock_timestamp() + make_interval(secs => $2::float8), + attempts = attempts + 1, + started_at = COALESCE(started_at, now()) +WHERE id = ( + SELECT id FROM hook_execution + WHERE (status = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= now())) + OR (status = 'running' AND lease_expires_at < now()) + ORDER BY created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +RETURNING id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at +` + +type ClaimOneHookExecutionParams struct { + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseTtlSeconds float64 `json:"lease_ttl_seconds"` +} + +// The executor's claim (MUL-4332 PR3 §7.2): lease one queued execution whose retry +// backoff has elapsed, or reclaim one abandoned by a crashed worker once its lease +// expires. Oldest first. The lease deadline comes from the DATABASE clock, the same +// clock every ownership predicate compares it against, so app/DB skew cannot grant an +// already-expired or over-long lease. FOR UPDATE SKIP LOCKED lets several executors +// share the queue. Rides idx_hook_execution_lease. +func (q *Queries) ClaimOneHookExecution(ctx context.Context, arg ClaimOneHookExecutionParams) (HookExecution, error) { + row := q.db.QueryRow(ctx, claimOneHookExecution, arg.LeaseToken, arg.LeaseTtlSeconds) + var i HookExecution + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ) + return i, err +} + +const createHook = `-- name: CreateHook :one + +INSERT INTO hook ( + id, workspace_id, name, enabled, active_revision_id, + scope_type, scope_id, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, + $9, $10, $11 +) +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type CreateHookParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` + Origin string `json:"origin"` + CreatorActorType string `json:"creator_actor_type"` + CreatorActorID pgtype.UUID `json:"creator_actor_id"` + AuthorizationPrincipalUserID pgtype.UUID `json:"authorization_principal_user_id"` +} + +// Event Hooks MVP (MUL-4332) — hook / revision / execution persistence access. +// All associations are application-validated; there are no foreign keys. Every +// write is workspace-scoped for tenant isolation. The hook and its immutable +// revisions are created together in one transaction with app-generated ids +// (hook.active_revision_id and hook_revision.id are chosen up front) because +// the two rows reference each other and there is no FK to order them. +func (q *Queries) CreateHook(ctx context.Context, arg CreateHookParams) (Hook, error) { + row := q.db.QueryRow(ctx, createHook, + arg.ID, + arg.WorkspaceID, + arg.Name, + arg.Enabled, + arg.ActiveRevisionID, + arg.ScopeType, + arg.ScopeID, + arg.Origin, + arg.CreatorActorType, + arg.CreatorActorID, + arg.AuthorizationPrincipalUserID, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const createHookActionEffect = `-- name: CreateHookActionEffect :one +INSERT INTO hook_action_effect ( + id, effect_key, execution_id, action_index, action_type, status, resolved_input, attempts +) VALUES ($1, $2, $3, $4, $5, 'running', $6, 1) +ON CONFLICT (effect_key) DO NOTHING +RETURNING id, effect_key, execution_id, action_index, action_type, status, resolved_input, output_type, output_id, attempts, error_code, error, created_at, completed_at +` + +type CreateHookActionEffectParams struct { + ID pgtype.UUID `json:"id"` + EffectKey string `json:"effect_key"` + ExecutionID pgtype.UUID `json:"execution_id"` + ActionIndex int32 `json:"action_index"` + ActionType string `json:"action_type"` + ResolvedInput []byte `json:"resolved_input"` +} + +// Claim one action's idempotency anchor. ON CONFLICT DO NOTHING means a concurrent or +// replayed attempt gets no row back and must read the existing effect instead of +// re-running the action. +func (q *Queries) CreateHookActionEffect(ctx context.Context, arg CreateHookActionEffectParams) (HookActionEffect, error) { + row := q.db.QueryRow(ctx, createHookActionEffect, + arg.ID, + arg.EffectKey, + arg.ExecutionID, + arg.ActionIndex, + arg.ActionType, + arg.ResolvedInput, + ) + var i HookActionEffect + err := row.Scan( + &i.ID, + &i.EffectKey, + &i.ExecutionID, + &i.ActionIndex, + &i.ActionType, + &i.Status, + &i.ResolvedInput, + &i.OutputType, + &i.OutputID, + &i.Attempts, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.CompletedAt, + ) + return i, err +} + +const createHookExecution = `-- name: CreateHookExecution :one +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, skip_reason, match_snapshot, condition_snapshot +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at +` + +type CreateHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + HookID pgtype.UUID `json:"hook_id"` + HookRevisionID pgtype.UUID `json:"hook_revision_id"` + EventID pgtype.UUID `json:"event_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + Status string `json:"status"` + SkipReason pgtype.Text `json:"skip_reason"` + MatchSnapshot []byte `json:"match_snapshot"` + ConditionSnapshot []byte `json:"condition_snapshot"` +} + +// Persist one matcher decision (queued to fire, or skipped with a reason) with +// the evaluator's structured snapshots and the pinned revision. Idempotent per +// (hook_id, event_id) via idx_hook_execution_hook_event, so re-processing a +// re-leased event never double-creates or double-advances the latch. +func (q *Queries) CreateHookExecution(ctx context.Context, arg CreateHookExecutionParams) (HookExecution, error) { + row := q.db.QueryRow(ctx, createHookExecution, + arg.ID, + arg.WorkspaceID, + arg.HookID, + arg.HookRevisionID, + arg.EventID, + arg.CorrelationID, + arg.Status, + arg.SkipReason, + arg.MatchSnapshot, + arg.ConditionSnapshot, + ) + var i HookExecution + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ) + return i, err +} + +const createHookExecutionFailure = `-- name: CreateHookExecutionFailure :one +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, error_code, error, completed_at +) VALUES ($1, $2, $3, $4, $5, $6, 'failed', $7, $8, now()) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at +` + +type CreateHookExecutionFailureParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + HookID pgtype.UUID `json:"hook_id"` + HookRevisionID pgtype.UUID `json:"hook_revision_id"` + EventID pgtype.UUID `json:"event_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` +} + +// Terminal isolation record for a candidate whose STORED CONFIG cannot be +// evaluated (a malformed revision fails identically on every retry). Writing it +// lets the matcher finish the remaining candidates and finalize the event, so one +// bad rule can never starve the healthy rules on the same event (MUL-4332 PR3 +// review round: matcher point 4). It carries no snapshots — evaluation never +// produced one. Same (hook_id, event_id) idempotency as the decision path. +func (q *Queries) CreateHookExecutionFailure(ctx context.Context, arg CreateHookExecutionFailureParams) (HookExecution, error) { + row := q.db.QueryRow(ctx, createHookExecutionFailure, + arg.ID, + arg.WorkspaceID, + arg.HookID, + arg.HookRevisionID, + arg.EventID, + arg.CorrelationID, + arg.ErrorCode, + arg.Error, + ) + var i HookExecution + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ) + return i, err +} + +const createHookRevision = `-- name: CreateHookRevision :one +INSERT INTO hook_revision ( + id, hook_id, revision, event_type, match, conditions, fire_mode, actions, + created_by_type, created_by_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, + $9, $10 +) +RETURNING id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at +` + +type CreateHookRevisionParams struct { + ID pgtype.UUID `json:"id"` + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` + EventType string `json:"event_type"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions []byte `json:"actions"` + CreatedByType string `json:"created_by_type"` + CreatedByID pgtype.UUID `json:"created_by_id"` +} + +func (q *Queries) CreateHookRevision(ctx context.Context, arg CreateHookRevisionParams) (HookRevision, error) { + row := q.db.QueryRow(ctx, createHookRevision, + arg.ID, + arg.HookID, + arg.Revision, + arg.EventType, + arg.Match, + arg.Conditions, + arg.FireMode, + arg.Actions, + arg.CreatedByType, + arg.CreatedByID, + ) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + +const deferExpiredHookExecution = `-- name: DeferExpiredHookExecution :execrows +UPDATE hook_execution +SET status = 'queued', + next_attempt_at = now() + make_interval(secs => $3::int), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 +` + +type DeferExpiredHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + BackoffSeconds int32 `json:"backoff_seconds"` +} + +// Back off an execution whose lease elapsed while THIS worker was running it. The +// CAS is on lease_token only — deliberately without the not-expired condition, since +// the lease is expired by definition here — so it fires only while the row still +// carries our token. If another worker has already reclaimed it the token differs, +// nothing is written, and the new owner is left alone. Without this the row keeps its +// original ordering position and the next claim selects it again immediately, +// starving everything behind it. +func (q *Queries) DeferExpiredHookExecution(ctx context.Context, arg DeferExpiredHookExecutionParams) (int64, error) { + result, err := q.db.Exec(ctx, deferExpiredHookExecution, arg.ID, arg.LeaseToken, arg.BackoffSeconds) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getHookActionEffect = `-- name: GetHookActionEffect :one +SELECT id, effect_key, execution_id, action_index, action_type, status, resolved_input, output_type, output_id, attempts, error_code, error, created_at, completed_at FROM hook_action_effect WHERE effect_key = $1 +` + +func (q *Queries) GetHookActionEffect(ctx context.Context, effectKey string) (HookActionEffect, error) { + row := q.db.QueryRow(ctx, getHookActionEffect, effectKey) + var i HookActionEffect + err := row.Scan( + &i.ID, + &i.EffectKey, + &i.ExecutionID, + &i.ActionIndex, + &i.ActionType, + &i.Status, + &i.ResolvedInput, + &i.OutputType, + &i.OutputID, + &i.Attempts, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.CompletedAt, + ) + return i, err +} + +const getHookForUpdate = `-- name: GetHookForUpdate :one +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE +` + +type GetHookForUpdateParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Row-locking load used by PATCH so concurrent edits to the same hook serialize: +// the lock holder allocates the next revision and repoints the active pointer +// before the next waiter reads MAX(revision), so idx_hook_revision_unique can +// never be violated by a MAX+1 race (MUL-4332 PR2 review point 4). +func (q *Queries) GetHookForUpdate(ctx context.Context, arg GetHookForUpdateParams) (Hook, error) { + row := q.db.QueryRow(ctx, getHookForUpdate, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const getHookInWorkspace = `-- name: GetHookInWorkspace :one +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE id = $1 AND workspace_id = $2 +` + +type GetHookInWorkspaceParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +func (q *Queries) GetHookInWorkspace(ctx context.Context, arg GetHookInWorkspaceParams) (Hook, error) { + row := q.db.QueryRow(ctx, getHookInWorkspace, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const getHookRevision = `-- name: GetHookRevision :one +SELECT id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at FROM hook_revision +WHERE id = $1 +` + +func (q *Queries) GetHookRevision(ctx context.Context, id pgtype.UUID) (HookRevision, error) { + row := q.db.QueryRow(ctx, getHookRevision, id) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + +const getHookRevisionByNumber = `-- name: GetHookRevisionByNumber :one +SELECT id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at FROM hook_revision +WHERE hook_id = $1 AND revision = $2 +` + +type GetHookRevisionByNumberParams struct { + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` +} + +// A specific revision of a hook, for `explain --revision N` (read-only debug). +func (q *Queries) GetHookRevisionByNumber(ctx context.Context, arg GetHookRevisionByNumberParams) (HookRevision, error) { + row := q.db.QueryRow(ctx, getHookRevisionByNumber, arg.HookID, arg.Revision) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + +const getMaxHookRevision = `-- name: GetMaxHookRevision :one +SELECT COALESCE(MAX(revision), 0)::int AS max_revision +FROM hook_revision +WHERE hook_id = $1 +` + +// Highest revision number for a hook, 0 when none exist yet. Used to compute the +// next revision on PATCH. +func (q *Queries) GetMaxHookRevision(ctx context.Context, hookID pgtype.UUID) (int32, error) { + row := q.db.QueryRow(ctx, getMaxHookRevision, hookID) + var max_revision int32 + err := row.Scan(&max_revision) + return max_revision, err +} + +const getOwnedHookExecution = `-- name: GetOwnedHookExecution :one +SELECT id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at FROM hook_execution +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +FOR UPDATE +` + +type GetOwnedHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +// Row-lock a claimed execution and assert lease OWNERSHIP before any action write. +// Identical predicate to every terminal write below, evaluated against database clock +// time, so a worker whose lease was reclaimed — or whose own lease elapsed — is not +// the owner and must write nothing (§7.3: a lost lease may never write terminal state). +func (q *Queries) GetOwnedHookExecution(ctx context.Context, arg GetOwnedHookExecutionParams) (HookExecution, error) { + row := q.db.QueryRow(ctx, getOwnedHookExecution, arg.ID, arg.LeaseToken) + var i HookExecution + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ) + return i, err +} + +const heartbeatHookExecution = `-- name: HeartbeatHookExecution :execrows +UPDATE hook_execution +SET lease_expires_at = clock_timestamp() + make_interval(secs => $3::float8) +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type HeartbeatHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseTtlSeconds float64 `json:"lease_ttl_seconds"` +} + +// Extend the lease of an execution this worker still owns (§7.2). A long action must +// not lose its lease simply because it is slow; the heartbeat keeps a live worker's +// claim valid while an ABANDONED claim still expires on schedule. +func (q *Queries) HeartbeatHookExecution(ctx context.Context, arg HeartbeatHookExecutionParams) (int64, error) { + result, err := q.db.Exec(ctx, heartbeatHookExecution, arg.ID, arg.LeaseToken, arg.LeaseTtlSeconds) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const listActiveHookRevisionsForEvent = `-- name: ListActiveHookRevisionsForEvent :many +SELECT + h.id AS hook_id, + r.id AS revision_id, + r.match AS match, + r.conditions AS conditions, + r.fire_mode AS fire_mode +FROM hook h +JOIN hook_revision r ON r.id = h.active_revision_id +WHERE h.workspace_id = $1 + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = $2 +ORDER BY h.created_at ASC, h.id ASC +` + +type ListActiveHookRevisionsForEventParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + EventType string `json:"event_type"` +} + +type ListActiveHookRevisionsForEventRow struct { + HookID pgtype.UUID `json:"hook_id"` + RevisionID pgtype.UUID `json:"revision_id"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` +} + +// Materialize the complete candidate set for a domain event in ONE statement: +// every enabled, non-archived hook in the workspace whose ACTIVE revision listens +// to this event type, together with that revision's full configuration. +// +// Being a SINGLE statement is the point (MUL-4332 PR3 review round: matcher point +// 1). Under READ COMMITTED every statement gets its own snapshot, so reading the +// candidate ids and then re-reading each hook's active_revision_id one by one could +// mix revisions from different instants within the same transaction. Returning +// (hook, revision) pairs from one statement pins the whole set at one instant, and +// the matcher calls this inside the transaction that claims the event, so the pin +// is taken at claim time. The matcher must NOT re-read active_revision_id +// afterwards — the pinned revision_id here is authoritative for this event. +// +// Issue scope is lifecycle ownership only and does NOT restrict the event subject — +// that is the job of `when` — so scope is not a filter here. +func (q *Queries) ListActiveHookRevisionsForEvent(ctx context.Context, arg ListActiveHookRevisionsForEventParams) ([]ListActiveHookRevisionsForEventRow, error) { + rows, err := q.db.Query(ctx, listActiveHookRevisionsForEvent, arg.WorkspaceID, arg.EventType) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListActiveHookRevisionsForEventRow{} + for rows.Next() { + var i ListActiveHookRevisionsForEventRow + if err := rows.Scan( + &i.HookID, + &i.RevisionID, + &i.Match, + &i.Conditions, + &i.FireMode, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listHookExecutionsByHook = `-- name: ListHookExecutionsByHook :many +SELECT id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at FROM hook_execution +WHERE hook_id = $1 +ORDER BY created_at DESC +LIMIT $2 +` + +type ListHookExecutionsByHookParams struct { + HookID pgtype.UUID `json:"hook_id"` + Limit int32 `json:"limit"` +} + +// Execution trace for the debug/explain endpoints. Newest first, bounded. +func (q *Queries) ListHookExecutionsByHook(ctx context.Context, arg ListHookExecutionsByHookParams) ([]HookExecution, error) { + rows, err := q.db.Query(ctx, listHookExecutionsByHook, arg.HookID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []HookExecution{} + for rows.Next() { + var i HookExecution + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listHooksByScope = `-- name: ListHooksByScope :many +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE workspace_id = $1 AND scope_type = $2 AND scope_id = $3 AND archived_at IS NULL +ORDER BY created_at DESC +` + +type ListHooksByScopeParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` +} + +func (q *Queries) ListHooksByScope(ctx context.Context, arg ListHooksByScopeParams) ([]Hook, error) { + rows, err := q.db.Query(ctx, listHooksByScope, arg.WorkspaceID, arg.ScopeType, arg.ScopeID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Hook{} + for rows.Next() { + var i Hook + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listHooksByWorkspace = `-- name: ListHooksByWorkspace :many +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE workspace_id = $1 AND archived_at IS NULL +ORDER BY created_at DESC +` + +func (q *Queries) ListHooksByWorkspace(ctx context.Context, workspaceID pgtype.UUID) ([]Hook, error) { + rows, err := q.db.Query(ctx, listHooksByWorkspace, workspaceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Hook{} + for rows.Next() { + var i Hook + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockHookForDecision = `-- name: LockHookForDecision :one +SELECT id FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE +` + +type LockHookForDecisionParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Serialize one hook's rising-edge latch read-modify-write against other matchers. +// This is a LOCK ONLY: the hook's active_revision_id is deliberately not returned, +// because the revision for this event was already pinned at claim time by +// ListActiveHookRevisionsForEvent and re-reading it here would reintroduce the +// drift that pin exists to prevent. +func (q *Queries) LockHookForDecision(ctx context.Context, arg LockHookForDecisionParams) (pgtype.UUID, error) { + row := q.db.QueryRow(ctx, lockHookForDecision, arg.ID, arg.WorkspaceID) + var id pgtype.UUID + err := row.Scan(&id) + return id, err +} + +const markHookActionEffectSucceeded = `-- name: MarkHookActionEffectSucceeded :execrows +UPDATE hook_action_effect +SET status = 'succeeded', output_type = $2, output_id = $3, completed_at = now() +WHERE effect_key = $1 +` + +type MarkHookActionEffectSucceededParams struct { + EffectKey string `json:"effect_key"` + OutputType pgtype.Text `json:"output_type"` + OutputID pgtype.UUID `json:"output_id"` +} + +func (q *Queries) MarkHookActionEffectSucceeded(ctx context.Context, arg MarkHookActionEffectSucceededParams) (int64, error) { + result, err := q.db.Exec(ctx, markHookActionEffectSucceeded, arg.EffectKey, arg.OutputType, arg.OutputID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const markHookExecutionFailed = `-- name: MarkHookExecutionFailed :execrows +UPDATE hook_execution +SET status = 'failed', error_code = $3, error = $4, completed_at = now(), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type MarkHookExecutionFailedParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` +} + +func (q *Queries) MarkHookExecutionFailed(ctx context.Context, arg MarkHookExecutionFailedParams) (int64, error) { + result, err := q.db.Exec(ctx, markHookExecutionFailed, + arg.ID, + arg.LeaseToken, + arg.ErrorCode, + arg.Error, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const markHookExecutionSkipped = `-- name: MarkHookExecutionSkipped :execrows +UPDATE hook_execution +SET status = 'skipped', skip_reason = $3, completed_at = now(), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type MarkHookExecutionSkippedParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + SkipReason pgtype.Text `json:"skip_reason"` +} + +// A terminal, non-retryable outcome (§7.3): permission, unavailable target, departed +// principal. Distinct from `failed`, which is an exhausted infrastructure retry. +func (q *Queries) MarkHookExecutionSkipped(ctx context.Context, arg MarkHookExecutionSkippedParams) (int64, error) { + result, err := q.db.Exec(ctx, markHookExecutionSkipped, arg.ID, arg.LeaseToken, arg.SkipReason) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const markHookExecutionSucceeded = `-- name: MarkHookExecutionSucceeded :execrows +UPDATE hook_execution +SET status = 'succeeded', completed_at = now(), lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type MarkHookExecutionSucceededParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +func (q *Queries) MarkHookExecutionSucceeded(ctx context.Context, arg MarkHookExecutionSucceededParams) (int64, error) { + result, err := q.db.Exec(ctx, markHookExecutionSucceeded, arg.ID, arg.LeaseToken) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const rescheduleHookExecution = `-- name: RescheduleHookExecution :execrows +UPDATE hook_execution +SET status = 'queued', + next_attempt_at = now() + make_interval(secs => $3::int), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +` + +type RescheduleHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` + BackoffSeconds int32 `json:"backoff_seconds"` +} + +// Release the lease and re-queue for a later attempt after an infrastructure failure. +// current_action_index is untouched, so the retry resumes at the action that failed +// and every action already committed stays committed (§7.2 partial execution). +func (q *Queries) RescheduleHookExecution(ctx context.Context, arg RescheduleHookExecutionParams) (int64, error) { + result, err := q.db.Exec(ctx, rescheduleHookExecution, arg.ID, arg.LeaseToken, arg.BackoffSeconds) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const setHookActiveRevision = `-- name: SetHookActiveRevision :one +UPDATE hook SET + active_revision_id = $3, + name = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type SetHookActiveRevisionParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + Name string `json:"name"` +} + +// Switch the active revision pointer and update the display name (PATCH). +// Revisions themselves are never mutated; a config change appends a new revision +// and repoints here. Scope is immutable after creation and is not touched. +func (q *Queries) SetHookActiveRevision(ctx context.Context, arg SetHookActiveRevisionParams) (Hook, error) { + row := q.db.QueryRow(ctx, setHookActiveRevision, + arg.ID, + arg.WorkspaceID, + arg.ActiveRevisionID, + arg.Name, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const setHookEnabled = `-- name: SetHookEnabled :one +UPDATE hook SET + enabled = $3, + disabled_reason = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type SetHookEnabledParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Enabled bool `json:"enabled"` + DisabledReason pgtype.Text `json:"disabled_reason"` +} + +func (q *Queries) SetHookEnabled(ctx context.Context, arg SetHookEnabledParams) (Hook, error) { + row := q.db.QueryRow(ctx, setHookEnabled, + arg.ID, + arg.WorkspaceID, + arg.Enabled, + arg.DisabledReason, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const upsertTerminalHookActionEffect = `-- name: UpsertTerminalHookActionEffect :exec +INSERT INTO hook_action_effect ( + id, effect_key, execution_id, action_index, action_type, + status, resolved_input, attempts, error_code, error, completed_at +) VALUES ($1, $2, $3, $4, $5, $7, $6, 1, $8, $9, now()) +ON CONFLICT (effect_key) DO UPDATE SET + status = EXCLUDED.status, + resolved_input = EXCLUDED.resolved_input, + attempts = hook_action_effect.attempts + 1, + error_code = EXCLUDED.error_code, + error = EXCLUDED.error, + completed_at = now() +` + +type UpsertTerminalHookActionEffectParams struct { + ID pgtype.UUID `json:"id"` + EffectKey string `json:"effect_key"` + ExecutionID pgtype.UUID `json:"execution_id"` + ActionIndex int32 `json:"action_index"` + ActionType string `json:"action_type"` + ResolvedInput []byte `json:"resolved_input"` + Status string `json:"status"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` +} + +// Record the durable audit row for an action that ended terminally — skipped or +// failed — carrying its resolved input and the reason. The success path writes its +// effect inside the action transaction, which rolls back on failure, so without this +// a skipped/failed action would leave no trace at all and a partial execution would +// show only the actions that succeeded. Keyed on effect_key so a retry updates the +// same row rather than creating a second one for the same (execution, action index). +func (q *Queries) UpsertTerminalHookActionEffect(ctx context.Context, arg UpsertTerminalHookActionEffectParams) error { + _, err := q.db.Exec(ctx, upsertTerminalHookActionEffect, + arg.ID, + arg.EffectKey, + arg.ExecutionID, + arg.ActionIndex, + arg.ActionType, + arg.ResolvedInput, + arg.Status, + arg.ErrorCode, + arg.Error, + ) + return err +} diff --git a/server/pkg/db/generated/issue.sql.go b/server/pkg/db/generated/issue.sql.go index 2c94c82b140..84392e10967 100644 --- a/server/pkg/db/generated/issue.sql.go +++ b/server/pkg/db/generated/issue.sql.go @@ -1159,6 +1159,64 @@ func (q *Queries) LockIssueDuplicateKey(ctx context.Context, dollar_1 string) er return err } +const lockIssueRowForUpdate = `-- name: LockIssueRowForUpdate :one +SELECT id, workspace_id, title, description, status, priority, assignee_type, assignee_id, creator_type, creator_id, parent_issue_id, acceptance_criteria, context_refs, position, due_date, created_at, updated_at, number, project_id, origin_type, origin_id, first_executed_at, start_date, metadata, stage, properties FROM issue +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE +` + +type LockIssueRowForUpdateParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Row-locks an issue and returns its full authoritative row for both a +// concurrency-safe update and domain-event emission (MUL-4332 review points 3 & +// 1). Callers read this INSIDE the update transaction, immediately before +// UpdateIssue/UpdateIssueStatus. Two purposes: +// 1. The event's `from` (status/assignee) reflects the truly-current row +// rather than a pre-tx snapshot, so concurrent transitions serialize on +// the lock and each records the correct edge instead of a stale `from`. +// 2. UpdateIssue writes the nullable columns (assignee, dates, parent, +// project, stage) as bare narg — an untouched field carries whatever the +// caller pre-filled. Pre-filling from a pre-tx snapshot lets a concurrent +// writer's change be silently rolled back; callers instead rebuild every +// untouched field from THIS locked row so an unrelated update never clobbers +// a field it did not intend to change. +func (q *Queries) LockIssueRowForUpdate(ctx context.Context, arg LockIssueRowForUpdateParams) (Issue, error) { + row := q.db.QueryRow(ctx, lockIssueRowForUpdate, arg.ID, arg.WorkspaceID) + var i Issue + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Title, + &i.Description, + &i.Status, + &i.Priority, + &i.AssigneeType, + &i.AssigneeID, + &i.CreatorType, + &i.CreatorID, + &i.ParentIssueID, + &i.AcceptanceCriteria, + &i.ContextRefs, + &i.Position, + &i.DueDate, + &i.CreatedAt, + &i.UpdatedAt, + &i.Number, + &i.ProjectID, + &i.OriginType, + &i.OriginID, + &i.FirstExecutedAt, + &i.StartDate, + &i.Metadata, + &i.Stage, + &i.Properties, + ) + return i, err +} + const markIssueFirstExecuted = `-- name: MarkIssueFirstExecuted :one UPDATE issue SET first_executed_at = now() diff --git a/server/pkg/db/generated/member.sql.go b/server/pkg/db/generated/member.sql.go index ee18f5e15ad..77f09499ae1 100644 --- a/server/pkg/db/generated/member.sql.go +++ b/server/pkg/db/generated/member.sql.go @@ -86,6 +86,64 @@ func (q *Queries) GetMemberByUserAndWorkspace(ctx context.Context, arg GetMember return i, err } +const getMemberByUserAndWorkspaceForShare = `-- name: GetMemberByUserAndWorkspaceForShare :one +SELECT id, workspace_id, user_id, role, created_at FROM member +WHERE user_id = $1 AND workspace_id = $2 +FOR SHARE +` + +type GetMemberByUserAndWorkspaceForShareParams struct { + UserID pgtype.UUID `json:"user_id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Locking membership read for authorization inside a write transaction. FOR SHARE +// takes a shared row lock that blocks a concurrent role UPDATE (demotion) or member +// DELETE (removal) from committing until this transaction ends, so a hook write can +// never commit under a membership/role that was revoked mid-transaction — a plain +// read under READ COMMITTED would miss that (MUL-4332 PR2 review round 5). Multiple +// concurrent hook writes may still share-lock the same member without blocking each +// other. Use ONLY inside the hook write transaction; the plain variant remains for +// non-transactional reads. +func (q *Queries) GetMemberByUserAndWorkspaceForShare(ctx context.Context, arg GetMemberByUserAndWorkspaceForShareParams) (Member, error) { + row := q.db.QueryRow(ctx, getMemberByUserAndWorkspaceForShare, arg.UserID, arg.WorkspaceID) + var i Member + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.UserID, + &i.Role, + &i.CreatedAt, + ) + return i, err +} + +const getMemberInWorkspace = `-- name: GetMemberInWorkspace :one +SELECT id, workspace_id, user_id, role, created_at FROM member +WHERE id = $1 AND workspace_id = $2 +` + +type GetMemberInWorkspaceParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Workspace-scoped member existence check by member row id. Used to fail-closed +// validate a send_inbox action target belongs to the hook's workspace +// (MUL-4332 PR2 review point 2). +func (q *Queries) GetMemberInWorkspace(ctx context.Context, arg GetMemberInWorkspaceParams) (Member, error) { + row := q.db.QueryRow(ctx, getMemberInWorkspace, arg.ID, arg.WorkspaceID) + var i Member + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.UserID, + &i.Role, + &i.CreatedAt, + ) + return i, err +} + const listMembers = `-- name: ListMembers :many SELECT id, workspace_id, user_id, role, created_at FROM member WHERE workspace_id = $1 @@ -167,6 +225,47 @@ func (q *Queries) ListMembersWithUser(ctx context.Context, workspaceID pgtype.UU return items, nil } +const listWorkspaceMembersByRoles = `-- name: ListWorkspaceMembersByRoles :many +SELECT id, workspace_id, user_id, role, created_at FROM member +WHERE workspace_id = $1 AND role = ANY($2::text[]) +ORDER BY created_at ASC +` + +type ListWorkspaceMembersByRolesParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + Roles []string `json:"roles"` +} + +// Members holding any of the given roles, for notifications that must reach the +// people able to act on them (e.g. an automation paused because its authorization +// principal left). Role filtering elsewhere is a per-caller authorization check; +// this is the fan-out selector, which did not previously exist. +func (q *Queries) ListWorkspaceMembersByRoles(ctx context.Context, arg ListWorkspaceMembersByRolesParams) ([]Member, error) { + rows, err := q.db.Query(ctx, listWorkspaceMembersByRoles, arg.WorkspaceID, arg.Roles) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Member{} + for rows.Next() { + var i Member + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.UserID, + &i.Role, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const updateMemberRole = `-- name: UpdateMemberRole :one UPDATE member SET role = $2 WHERE id = $1 diff --git a/server/pkg/db/generated/models.go b/server/pkg/db/generated/models.go index b7f277e5409..1a75488f0c2 100644 --- a/server/pkg/db/generated/models.go +++ b/server/pkg/db/generated/models.go @@ -173,6 +173,16 @@ type Attachment struct { TaskID pgtype.UUID `json:"task_id"` } +// Durable row-lockable automation state, not an audit log. No foreign keys. +type AutomationState struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` + State []byte `json:"state"` + Version int64 `json:"version"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` +} + type Autopilot struct { ID pgtype.UUID `json:"id"` WorkspaceID pgtype.UUID `json:"workspace_id"` @@ -452,6 +462,31 @@ type DaemonToken struct { CreatedAt pgtype.Timestamptz `json:"created_at"` } +// Transactional-outbox domain event log (MUL-4332 §4.1). One row per committed domain fact, written in the same tx as the fact. Source of truth for the hooks engine; no FK, no cascade, app-layer integrity. dispatch_* columns are the single-consumer outbox lease consumed by the PR3 matcher. +type DomainEvent struct { + ID pgtype.UUID `json:"id"` + Seq int64 `json:"seq"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID pgtype.UUID `json:"actor_id"` + Payload []byte `json:"payload"` + CorrelationID pgtype.UUID `json:"correlation_id"` + CausationExecutionID pgtype.UUID `json:"causation_execution_id"` + CausationActionIndex pgtype.Int4 `json:"causation_action_index"` + HopCount int32 `json:"hop_count"` + DispatchStatus string `json:"dispatch_status"` + Attempts int32 `json:"attempts"` + AvailableAt pgtype.Timestamptz `json:"available_at"` + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseExpiresAt pgtype.Timestamptz `json:"lease_expires_at"` + DispatchedAt pgtype.Timestamptz `json:"dispatched_at"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type Feedback struct { ID pgtype.UUID `json:"id"` UserID pgtype.UUID `json:"user_id"` @@ -533,6 +568,85 @@ type GithubPullRequestCheckSuite struct { UpdatedAt pgtype.Timestamptz `json:"updated_at"` } +// Event Hooks stable identity and lifecycle owner. No foreign keys; application validates all associations. +type Hook struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` + RetireAfterEventSeq pgtype.Int8 `json:"retire_after_event_seq"` + Origin string `json:"origin"` + SystemKey pgtype.Text `json:"system_key"` + SystemVersion pgtype.Int4 `json:"system_version"` + CreatorActorType string `json:"creator_actor_type"` + CreatorActorID pgtype.UUID `json:"creator_actor_id"` + AuthorizationPrincipalUserID pgtype.UUID `json:"authorization_principal_user_id"` + DisabledReason pgtype.Text `json:"disabled_reason"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + ArchivedAt pgtype.Timestamptz `json:"archived_at"` +} + +// One durable idempotency record per hook execution action. No foreign keys. +type HookActionEffect struct { + ID pgtype.UUID `json:"id"` + EffectKey string `json:"effect_key"` + ExecutionID pgtype.UUID `json:"execution_id"` + ActionIndex int32 `json:"action_index"` + ActionType string `json:"action_type"` + Status string `json:"status"` + ResolvedInput []byte `json:"resolved_input"` + OutputType pgtype.Text `json:"output_type"` + OutputID pgtype.UUID `json:"output_id"` + Attempts int32 `json:"attempts"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + CompletedAt pgtype.Timestamptz `json:"completed_at"` +} + +// Durable matcher/executor trace with revision pinning. No foreign keys; application validates all associations. +type HookExecution struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + HookID pgtype.UUID `json:"hook_id"` + HookRevisionID pgtype.UUID `json:"hook_revision_id"` + EventID pgtype.UUID `json:"event_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + Status string `json:"status"` + SkipReason pgtype.Text `json:"skip_reason"` + MatchSnapshot []byte `json:"match_snapshot"` + ConditionSnapshot []byte `json:"condition_snapshot"` + CurrentActionIndex int32 `json:"current_action_index"` + Attempts int32 `json:"attempts"` + NextAttemptAt pgtype.Timestamptz `json:"next_attempt_at"` + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseExpiresAt pgtype.Timestamptz `json:"lease_expires_at"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + StartedAt pgtype.Timestamptz `json:"started_at"` + CompletedAt pgtype.Timestamptz `json:"completed_at"` +} + +// Immutable Event Hooks configuration revisions. No foreign keys; application validates hook ownership. +type HookRevision struct { + ID pgtype.UUID `json:"id"` + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` + EventType string `json:"event_type"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions []byte `json:"actions"` + CreatedByType string `json:"created_by_type"` + CreatedByID pgtype.UUID `json:"created_by_id"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type InboxItem struct { ID pgtype.UUID `json:"id"` WorkspaceID pgtype.UUID `json:"workspace_id"` diff --git a/server/pkg/db/generated/runtime.sql.go b/server/pkg/db/generated/runtime.sql.go index e2999d62d18..18c2e3734e7 100644 --- a/server/pkg/db/generated/runtime.sql.go +++ b/server/pkg/db/generated/runtime.sql.go @@ -217,89 +217,6 @@ func (q *Queries) DeleteSystemAgentsByRuntime(ctx context.Context, runtimeID pgt return err } -const failTasksForOfflineRuntimes = `-- name: FailTasksForOfflineRuntimes :many -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'runtime went offline', - failure_reason = 'runtime_offline', - wait_reason = NULL -WHERE status IN ('dispatched', 'running', 'waiting_local_directory') - AND runtime_id IN ( - SELECT id FROM agent_runtime WHERE status = 'offline' - ) -RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id -` - -// Marks dispatched/running/waiting_local_directory tasks as failed when -// their runtime is offline. This cleans up orphaned tasks after a daemon -// crash or network partition. -func (q *Queries) FailTasksForOfflineRuntimes(ctx context.Context) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, failTasksForOfflineRuntimes) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const findLegacyRuntimesByDaemonID = `-- name: FindLegacyRuntimesByDaemonID :many SELECT id, workspace_id, daemon_id, name, runtime_mode, provider, status, device_info, metadata, last_seen_at, created_at, updated_at, owner_id, legacy_daemon_id, visibility, profile_id, custom_name FROM agent_runtime WHERE workspace_id = $1 @@ -950,6 +867,94 @@ func (q *Queries) SelectStaleOnlineRuntimes(ctx context.Context, staleSeconds fl return items, nil } +const selectTasksForOfflineRuntimes = `-- name: SelectTasksForOfflineRuntimes :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE status IN ('dispatched', 'running', 'waiting_local_directory') + AND runtime_id IN ( + SELECT id FROM agent_runtime WHERE status = 'offline' + ) +ORDER BY created_at ASC +LIMIT $1::int +FOR UPDATE SKIP LOCKED +` + +// Selects (and row-locks) up to @max_per_tick dispatched/running/ +// waiting_local_directory tasks whose runtime is offline — the orphans a daemon +// crash or network partition leaves behind. This is the candidate half of the +// MUL-4332 poison-isolated fail path (review point 2): the caller resolves each +// task's workspace, fails only the resolvable set via FailAgentTasksByIDs and +// emits its task.failed event in the SAME transaction, so an unresolvable poison +// row is skipped instead of rolling back the whole batch. FOR UPDATE SKIP LOCKED +// keeps the sweeper off rows a daemon is actively claiming; the LIMIT bounds the +// lock hold and the rest drain on later ticks. +func (q *Queries) SelectTasksForOfflineRuntimes(ctx context.Context, maxPerTick int32) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectTasksForOfflineRuntimes, maxPerTick) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const setAgentRuntimeOffline = `-- name: SetAgentRuntimeOffline :exec UPDATE agent_runtime SET status = 'offline', updated_at = now() diff --git a/server/pkg/db/queries/agent.sql b/server/pkg/db/queries/agent.sql index e817ebf8b70..b5fd1eae25f 100644 --- a/server/pkg/db/queries/agent.sql +++ b/server/pkg/db/queries/agent.sql @@ -698,25 +698,61 @@ SET session_id = COALESCE(sqlc.narg('session_id'), session_id), work_dir = COALESCE(sqlc.narg('work_dir'), work_dir) WHERE id = $1 AND status IN ('dispatched', 'running'); --- name: RecoverOrphanedTasksForRuntime :many --- Called by the daemon at startup. Atomically fails any dispatched/running/ --- waiting_local_directory task that the prior incarnation of this runtime --- owned but did not finalize. Returns the failed rows so callers can hand --- them to the auto-retry path. waiting_local_directory rows are included --- because the daemon holding the path lock is the same process that just --- died — without us, the row would sit waiting forever. -UPDATE agent_task_queue -SET status = 'failed', - completed_at = now(), - error = 'daemon restarted while task was in flight', - failure_reason = 'runtime_recovery', - wait_reason = NULL, - prepare_lease_expires_at = NULL -WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') -RETURNING *; +-- name: SelectOrphanedTasksForRuntime :many +-- Called by the daemon at startup. Selects (and row-locks) up to @max_per_tick +-- dispatched/running/waiting_local_directory tasks that the prior incarnation of +-- this runtime owned but did not finalize. waiting_local_directory rows are +-- included because the daemon holding the path lock is the same process that +-- just died — without us, the row would sit waiting forever. +-- +-- Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +-- caller resolves each task's workspace, fails only the resolvable set via +-- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +-- The LIMIT bounds the lock hold. +-- +-- Plain FOR UPDATE (NOT SKIP LOCKED — review round 4, point 1). Unlike the +-- sweepers, which re-scan from the start every tick so a skipped row is simply +-- retried next tick, this query pages forward with a keyset cursor that advances +-- permanently past whatever a page returned. If SKIP LOCKED silently dropped an +-- older orphan that happened to be briefly locked (a sweep, a stale-dispatch +-- reclaim), the cursor — filled by the newer rows behind it — would step past that +-- older row and NO later page could ever select it again; the runtime is already +-- back `online`, so the offline sweep won't reap it either, and it leaks forever. +-- Plain FOR UPDATE instead WAITS for the lock and, once it releases, re-checks the +-- row against the WHERE (Postgres EvalPlanQual): still dispatched/running → +-- included on this page; already failed by whoever held the lock → correctly +-- excluded. The bounded page size keeps that wait short, and recovery only races +-- short sweeper/reclaim transactions (both SKIP LOCKED, so they never wait — no +-- deadlock cycle is possible). +-- +-- Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts +-- the runtime back to `online`, so the every-tick offline sweep will NOT reap an +-- orphan the daemon leaves past this page — the recovery must therefore drain +-- itself. Callers page by (created_at, id) via @after_created_at / @after_id (NULL +-- on the first page) and repeat until a short page. Because the cursor advances +-- over EVERY returned candidate — including a poison (unresolvable-workspace) row +-- the caller skips rather than fails — a page full of poison at the front can no +-- longer pin the drain in place: the next page steps past it to the healthy rows +-- behind it. (A plain re-select of the oldest rows would loop on the poison +-- forever.) Ordering matches the keyset so pages are stable and non-overlapping. +SELECT * FROM agent_task_queue +WHERE runtime_id = @runtime_id AND status IN ('dispatched', 'running', 'waiting_local_directory') + AND ( + sqlc.narg('after_created_at')::timestamptz IS NULL + OR created_at > sqlc.narg('after_created_at')::timestamptz + OR (created_at = sqlc.narg('after_created_at')::timestamptz AND id > sqlc.narg('after_id')::uuid) + ) +ORDER BY created_at ASC, id ASC +LIMIT @max_per_tick::int +FOR UPDATE; --- name: FailStaleTasks :many --- Fails tasks stuck in dispatched/running beyond the given thresholds. +-- name: SelectStaleTasksToFail :many +-- Selects (and row-locks) up to @max_per_tick tasks stuck in dispatched/running +-- beyond the given thresholds. Candidate half of the MUL-4332 poison-isolated +-- fail path (review point 2): the caller resolves each task's workspace, fails +-- only the resolvable set via FailAgentTasksByIDs and emits its task.failed event +-- in the SAME transaction. FOR UPDATE SKIP LOCKED keeps the backstop off rows a +-- daemon is actively claiming; the LIMIT bounds the lock hold. -- -- Each branch pairs a wall-clock deadline with a task-appropriate liveness -- signal, so the sweeper only kills tasks whose owning daemon is no longer @@ -738,8 +774,9 @@ RETURNING *; -- server-side wall clock must not shadow that with a coarser cap. -- -- The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` --- (which mixes DB `last_seen_at` with the Redis LivenessStore and calls --- `FailTasksForOfflineRuntimes` in the same tick). The wall-clock branch +-- (which mixes DB `last_seen_at` with the Redis LivenessStore and fails +-- offline-runtime tasks via SelectTasksForOfflineRuntimes in the same tick). +-- The wall-clock branch -- here is a defensive backstop for pathological cases where a runtime row -- somehow retains status='online' with a stale DB heartbeat for longer than -- the wall clock allows. @@ -751,12 +788,9 @@ RETURNING *; -- waiting_local_directory rows are intentionally excluded: the daemon owns -- the wait (with its own ctx-driven timeout) and a legitimate queue ahead -- of this task can exceed the dispatch / running timeouts without being --- "stuck". If the daemon dies, RecoverOrphanedTasksForRuntime reclaims +-- "stuck". If the daemon dies, SelectOrphanedTasksForRuntime reclaims -- those rows at restart. -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'task timed out', - failure_reason = 'timeout', - prepare_lease_expires_at = NULL +SELECT * FROM agent_task_queue WHERE ( status = 'dispatched' AND dispatched_at < now() - make_interval(secs => @dispatch_timeout_secs::double precision) @@ -772,50 +806,50 @@ WHERE ( AND r.last_seen_at >= now() - make_interval(secs => @runtime_stale_secs::double precision) ) ) -RETURNING *; - --- name: ExpireStaleQueuedTasks :many --- Fails tasks that have been sitting in 'queued' for longer than the TTL. --- This is the cleanup arm of the MUL-1899 "queued backlog" fix: even with the --- new dispatch-time admission gate that refuses to enqueue when the runtime --- is offline, we still need to drain the historical 87k+ doomed rows and --- handle edge cases where a runtime goes offline AFTER a task is already --- queued (the admission check protects new enqueues, not in-flight queue --- depth). +ORDER BY started_at ASC NULLS FIRST, dispatched_at ASC NULLS FIRST +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; + +-- name: SelectExpiredQueuedTasks :many +-- Selects (and row-locks) up to @max_per_tick tasks sitting in 'queued' past the +-- TTL — the cleanup arm of the MUL-1899 "queued backlog" fix (drains the +-- historical 87k+ doomed rows and catches the case where a runtime goes offline +-- AFTER a task is already queued). -- --- Concurrency safety: the daemon's claim path may race with this sweeper to --- transition the same row out of 'queued'. We protect against that two --- ways: --- 1. The CTE selects victims with FOR UPDATE SKIP LOCKED so a row that is --- currently being claimed (or otherwise locked) is skipped — no lock --- contention with the dispatch path, and we won't queue up behind it. --- 2. The outer UPDATE re-checks status='queued' AND the TTL predicate at --- apply time. If a daemon claimed the row between selection and update --- (e.g. lock released after the claim transaction commits), the row is --- already 'dispatched'/'running' and the WHERE clause filters it out --- so we cannot clobber an in-flight task. --- Capped via LIMIT inside the CTE so a single sweep tick cannot monopolise --- the DB when the backlog is large — the sweeper drains the rest on --- subsequent ticks. -WITH victims AS ( - SELECT id FROM agent_task_queue - WHERE status = 'queued' - AND created_at < now() - make_interval(secs => @ttl_secs::double precision) - ORDER BY created_at ASC - LIMIT @max_per_tick::int - FOR UPDATE SKIP LOCKED -) -UPDATE agent_task_queue t +-- Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +-- caller resolves each task's workspace, fails only the resolvable set via +-- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +-- FOR UPDATE SKIP LOCKED skips rows a daemon is currently claiming, so we never +-- contend with the dispatch path or clobber an in-flight task; because the fail +-- runs in this same transaction the locked rows cannot transition out of 'queued' +-- underneath us. The LIMIT bounds the lock hold; the rest drain on later ticks. +SELECT * FROM agent_task_queue +WHERE status = 'queued' + AND created_at < now() - make_interval(secs => @ttl_secs::double precision) +ORDER BY created_at ASC +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; + +-- name: FailAgentTasksByIDs :many +-- Fails a specific, already-resolved set of tasks by id — the terminal half of +-- the MUL-4332 poison-isolated bulk fail (review point 2). The caller has just +-- selected these ids with SelectStaleTasksToFail / SelectExpiredQueuedTasks / +-- SelectTasksForOfflineRuntimes / SelectOrphanedTasksForRuntime FOR UPDATE in +-- the SAME transaction, so the rows are locked and cannot have transitioned; +-- the status guard is a defensive backstop that keeps this idempotent if the id +-- set is ever reused. @error / @failure_reason are supplied per sweeper. Both +-- wait_reason and prepare_lease_expires_at are cleared (clearing a NULL is a +-- no-op) so this one query serves every bulk-fail caller. +UPDATE agent_task_queue SET status = 'failed', completed_at = now(), - error = 'task expired in queue', - failure_reason = 'queued_expired', + error = @error, + failure_reason = @failure_reason, + wait_reason = NULL, prepare_lease_expires_at = NULL -FROM victims v -WHERE t.id = v.id - AND t.status = 'queued' - AND t.created_at < now() - make_interval(secs => @ttl_secs::double precision) -RETURNING t.*; +WHERE id = ANY(@ids::uuid[]) + AND status IN ('queued', 'dispatched', 'running', 'waiting_local_directory') +RETURNING *; -- name: CancelAgentTask :one UPDATE agent_task_queue diff --git a/server/pkg/db/queries/automation_state.sql b/server/pkg/db/queries/automation_state.sql new file mode 100644 index 00000000000..6329f0c4ae4 --- /dev/null +++ b/server/pkg/db/queries/automation_state.sql @@ -0,0 +1,16 @@ +-- Durable row-lockable automation state (MUL-4332): rising-edge latches (and, +-- later, stage frontier + rate buckets). Not an audit log; one row per key. + +-- name: GetAutomationStateForUpdate :one +-- Row-lock a state key so concurrent matchers serialize their read-modify-write +-- of the same latch. Returns no rows the first time a key is used. +SELECT * FROM automation_state +WHERE workspace_id = $1 AND state_kind = $2 AND state_key = $3 +FOR UPDATE; + +-- name: UpsertAutomationState :one +INSERT INTO automation_state (workspace_id, state_kind, state_key, state, version) +VALUES ($1, $2, $3, $4, 0) +ON CONFLICT (workspace_id, state_kind, state_key) +DO UPDATE SET state = EXCLUDED.state, version = automation_state.version + 1, updated_at = now() +RETURNING *; diff --git a/server/pkg/db/queries/domain_event.sql b/server/pkg/db/queries/domain_event.sql new file mode 100644 index 00000000000..44fb686cc0d --- /dev/null +++ b/server/pkg/db/queries/domain_event.sql @@ -0,0 +1,184 @@ +-- Transactional-outbox domain event log (MUL-4332 §4.1). CreateDomainEvent is +-- the only write; callers invoke it on a *db.Queries already bound to their +-- own pgx.Tx (WithTx) so the event commits atomically with the domain fact. + +-- name: CreateDomainEvent :one +-- dispatch_status ('pending'), available_at (now()), attempts (0), seq +-- (nextval) and created_at (now()) all come from column defaults — this is a +-- root outbox write, never a re-queue, so the writer never sets them. +INSERT INTO domain_event ( + id, + workspace_id, + type, + schema_version, + subject_type, + subject_id, + actor_type, + actor_id, + payload, + correlation_id, + causation_execution_id, + causation_action_index, + hop_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 +) +RETURNING *; + +-- name: GetDomainEvent :one +SELECT * FROM domain_event +WHERE id = $1; + +-- name: ListDomainEventsByCorrelation :many +-- Bounded correlation-chain read for the debug API. The LIMIT is pushed into the +-- query (not applied after loading the whole chain) and rides the +-- (workspace_id, correlation_id, seq) index (MUL-4332 PR3 review round: correlation). +SELECT * FROM domain_event +WHERE workspace_id = $1 + AND correlation_id = $2 +ORDER BY seq ASC +LIMIT $3; + +-- name: CountDomainEventsBySubject :one +SELECT count(*) FROM domain_event +WHERE subject_type = $1 + AND subject_id = $2; + +-- NOTE: the retention/TTL delete is intentionally NOT defined in PR1. The +-- correct predicate is "dispatched AND older than TTL AND every related +-- hook_execution is terminal" (MUL-4332 §4.1/§9), and hook_execution does not +-- exist until PR3. Shipping a weaker "dispatched + TTL" delete now would risk +-- reclaiming still-executing audit sources the moment PR3 enables dispatching +-- (review point 5). The query lands in PR3 with the full terminal predicate. + +-- name: ClaimOneEventWithCandidates :many +-- The durable matcher's claim AND revision pin, in a SINGLE statement (MUL-4332 PR3 +-- review round: matcher point 1). It leases exactly one undispatched, now-available +-- event in seq order and, in the same statement, materializes that event's complete +-- (hook_id, revision_id) candidate set with each revision's configuration. +-- +-- One statement is what makes the pin real. PostgreSQL's default READ COMMITTED +-- gives every statement its own snapshot, so claiming in one statement and then +-- selecting candidates in another — even inside one transaction — lets a revision +-- edit committed in between change this event's decision, and lets two candidates be +-- decided against revisions from different instants. Here the claim and the whole +-- candidate set share one snapshot, so the pinned revisions are exactly those active +-- at claim time (§5.1 "使用 matcher claim 时的当前 enabled revision"). +-- +-- The LEFT JOIN LATERAL means an event with no candidates still returns exactly one +-- row (hook_id NULL), so the caller can distinguish "claimed, nothing matched" from +-- "nothing to claim" (zero rows). Reclaims events left 'dispatching' by a crashed +-- matcher once their lease expires, so processing is at-least-once. FOR UPDATE SKIP +-- LOCKED lets multiple matchers share the queue and keeps the claimed row locked for +-- the rest of the transaction. Rides idx_domain_event_dispatch. +-- +-- Issue scope is lifecycle ownership only and does NOT restrict the event subject — +-- that is the job of `when` — so scope is not a filter here. +WITH claimed AS ( + UPDATE domain_event + SET dispatch_status = 'dispatching', + lease_token = @lease_token, + -- Derive the deadline from the DATABASE clock, the same clock the ownership + -- predicate compares it against. Computing it application-side would let + -- app/DB clock skew grant a lease that is already expired, or one that + -- outlives its intended TTL. + lease_expires_at = clock_timestamp() + make_interval(secs => @lease_ttl_seconds::float8), + attempts = attempts + 1 + WHERE id = ( + SELECT id FROM domain_event + WHERE available_at <= now() + AND (dispatch_status = 'pending' + OR (dispatch_status = 'dispatching' AND lease_expires_at < now())) + ORDER BY seq ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, workspace_id, type, subject_id, actor_type, actor_id, + payload, correlation_id, hop_count +) +SELECT + c.id AS event_id, + c.workspace_id AS event_workspace_id, + c.type AS event_type, + c.subject_id AS event_subject_id, + c.actor_type AS event_actor_type, + c.actor_id AS event_actor_id, + c.payload AS event_payload, + c.correlation_id AS event_correlation_id, + c.hop_count AS event_hop_count, + cand.hook_id AS hook_id, + cand.revision_id AS revision_id, + cand.match AS match, + cand.conditions AS conditions, + -- COALESCE so the no-candidate row (every cand.* NULL) still scans into a + -- non-nullable string; callers gate on hook_id being present. + COALESCE(cand.fire_mode, '')::text AS fire_mode +FROM claimed c +LEFT JOIN LATERAL ( + SELECT h.id AS hook_id, h.created_at AS hook_created_at, + r.id AS revision_id, r.match, r.conditions, r.fire_mode + FROM hook h + JOIN hook_revision r ON r.id = h.active_revision_id + WHERE h.workspace_id = c.workspace_id + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = c.type +) cand ON true +ORDER BY cand.hook_created_at ASC NULLS LAST, cand.hook_id ASC; + +-- name: DeferDomainEventDispatch :execrows +-- Back off one event after a transient dispatch failure. The failed decision rolled +-- back (including its claim), so without this the event would sit at the head of the +-- queue and be re-claimed immediately, spinning on the same failure and starving +-- everything behind it. +UPDATE domain_event +SET available_at = now() + make_interval(secs => @backoff_seconds::int), + attempts = attempts + 1 +WHERE id = $1; + +-- name: GetOwnedDomainEventForDispatch :one +-- Row-lock one claimed event and assert lease OWNERSHIP before the matcher writes +-- any decision (MUL-4332 PR3 review round: matcher point 2). The predicate is +-- deliberately identical to the one MarkDomainEventDispatched/Failed use, and both +-- evaluate the expiry against DATABASE clock time (clock_timestamp(), not now(), +-- which is frozen at transaction start): a worker holding the right token whose +-- lease has nonetheless expired is NOT the owner and must write nothing. Returning +-- no rows is the fail-closed signal. +SELECT * FROM domain_event +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +FOR UPDATE; + +-- name: MarkDomainEventDispatched :execrows +-- Finalize a matched event under the SAME ownership predicate as the entry +-- assertion, so a lease that expired mid-decision cannot commit the decision. The +-- caller MUST assert this returns exactly 1 — a 0 means ownership was lost and the +-- whole transaction (every execution and latch it wrote) must roll back. +-- dispatched_at is set here (and only here) so the retention/audit boundary can +-- rely on "dispatched ⇒ dispatched_at". +UPDATE domain_event +SET dispatch_status = 'dispatched', + dispatched_at = now(), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: MarkDomainEventFailed :execrows +-- Terminal failure for an event the matcher can never decode (a malformed payload +-- fails identically on every retry). Recording it 'failed' instead of leaving it +-- pending stops one poison event from being re-leased forever. Same ownership +-- predicate as the dispatched path. +UPDATE domain_event +SET dispatch_status = 'failed', + dispatched_at = now(), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND dispatch_status = 'dispatching' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); diff --git a/server/pkg/db/queries/hook.sql b/server/pkg/db/queries/hook.sql new file mode 100644 index 00000000000..85a677d48e8 --- /dev/null +++ b/server/pkg/db/queries/hook.sql @@ -0,0 +1,321 @@ +-- Event Hooks MVP (MUL-4332) — hook / revision / execution persistence access. +-- All associations are application-validated; there are no foreign keys. Every +-- write is workspace-scoped for tenant isolation. The hook and its immutable +-- revisions are created together in one transaction with app-generated ids +-- (hook.active_revision_id and hook_revision.id are chosen up front) because +-- the two rows reference each other and there is no FK to order them. + +-- name: CreateHook :one +INSERT INTO hook ( + id, workspace_id, name, enabled, active_revision_id, + scope_type, scope_id, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, + $9, $10, $11 +) +RETURNING *; + +-- name: GetHookInWorkspace :one +SELECT * FROM hook +WHERE id = $1 AND workspace_id = $2; + +-- name: GetHookForUpdate :one +-- Row-locking load used by PATCH so concurrent edits to the same hook serialize: +-- the lock holder allocates the next revision and repoints the active pointer +-- before the next waiter reads MAX(revision), so idx_hook_revision_unique can +-- never be violated by a MAX+1 race (MUL-4332 PR2 review point 4). +SELECT * FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE; + +-- name: ListHooksByWorkspace :many +SELECT * FROM hook +WHERE workspace_id = $1 AND archived_at IS NULL +ORDER BY created_at DESC; + +-- name: ListHooksByScope :many +SELECT * FROM hook +WHERE workspace_id = $1 AND scope_type = $2 AND scope_id = $3 AND archived_at IS NULL +ORDER BY created_at DESC; + +-- name: SetHookActiveRevision :one +-- Switch the active revision pointer and update the display name (PATCH). +-- Revisions themselves are never mutated; a config change appends a new revision +-- and repoints here. Scope is immutable after creation and is not touched. +UPDATE hook SET + active_revision_id = $3, + name = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: SetHookEnabled :one +UPDATE hook SET + enabled = $3, + disabled_reason = sqlc.narg('disabled_reason'), + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: ArchiveHook :one +-- Soft archive (DELETE). Existing revisions / executions / effects are retained +-- for audit; the hook is also disabled so nothing can match it. +UPDATE hook SET + archived_at = now(), + enabled = false, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: CreateHookRevision :one +INSERT INTO hook_revision ( + id, hook_id, revision, event_type, match, conditions, fire_mode, actions, + created_by_type, created_by_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, + $9, $10 +) +RETURNING *; + +-- name: GetHookRevision :one +SELECT * FROM hook_revision +WHERE id = $1; + +-- name: GetHookRevisionByNumber :one +-- A specific revision of a hook, for `explain --revision N` (read-only debug). +SELECT * FROM hook_revision +WHERE hook_id = $1 AND revision = $2; + +-- name: GetMaxHookRevision :one +-- Highest revision number for a hook, 0 when none exist yet. Used to compute the +-- next revision on PATCH. +SELECT COALESCE(MAX(revision), 0)::int AS max_revision +FROM hook_revision +WHERE hook_id = $1; + +-- name: ListHookExecutionsByHook :many +-- Execution trace for the debug/explain endpoints. Newest first, bounded. +SELECT * FROM hook_execution +WHERE hook_id = $1 +ORDER BY created_at DESC +LIMIT $2; + +-- name: ListActiveHookRevisionsForEvent :many +-- Materialize the complete candidate set for a domain event in ONE statement: +-- every enabled, non-archived hook in the workspace whose ACTIVE revision listens +-- to this event type, together with that revision's full configuration. +-- +-- Being a SINGLE statement is the point (MUL-4332 PR3 review round: matcher point +-- 1). Under READ COMMITTED every statement gets its own snapshot, so reading the +-- candidate ids and then re-reading each hook's active_revision_id one by one could +-- mix revisions from different instants within the same transaction. Returning +-- (hook, revision) pairs from one statement pins the whole set at one instant, and +-- the matcher calls this inside the transaction that claims the event, so the pin +-- is taken at claim time. The matcher must NOT re-read active_revision_id +-- afterwards — the pinned revision_id here is authoritative for this event. +-- +-- Issue scope is lifecycle ownership only and does NOT restrict the event subject — +-- that is the job of `when` — so scope is not a filter here. +SELECT + h.id AS hook_id, + r.id AS revision_id, + r.match AS match, + r.conditions AS conditions, + r.fire_mode AS fire_mode +FROM hook h +JOIN hook_revision r ON r.id = h.active_revision_id +WHERE h.workspace_id = $1 + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = $2 +ORDER BY h.created_at ASC, h.id ASC; + +-- name: LockHookForDecision :one +-- Serialize one hook's rising-edge latch read-modify-write against other matchers. +-- This is a LOCK ONLY: the hook's active_revision_id is deliberately not returned, +-- because the revision for this event was already pinned at claim time by +-- ListActiveHookRevisionsForEvent and re-reading it here would reintroduce the +-- drift that pin exists to prevent. +SELECT id FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE; + +-- name: CreateHookExecution :one +-- Persist one matcher decision (queued to fire, or skipped with a reason) with +-- the evaluator's structured snapshots and the pinned revision. Idempotent per +-- (hook_id, event_id) via idx_hook_execution_hook_event, so re-processing a +-- re-leased event never double-creates or double-advances the latch. +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, skip_reason, match_snapshot, condition_snapshot +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING *; + +-- name: CreateHookExecutionFailure :one +-- Terminal isolation record for a candidate whose STORED CONFIG cannot be +-- evaluated (a malformed revision fails identically on every retry). Writing it +-- lets the matcher finish the remaining candidates and finalize the event, so one +-- bad rule can never starve the healthy rules on the same event (MUL-4332 PR3 +-- review round: matcher point 4). It carries no snapshots — evaluation never +-- produced one. Same (hook_id, event_id) idempotency as the decision path. +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, error_code, error, completed_at +) VALUES ($1, $2, $3, $4, $5, $6, 'failed', $7, $8, now()) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING *; + +-- name: ClaimOneHookExecution :one +-- The executor's claim (MUL-4332 PR3 §7.2): lease one queued execution whose retry +-- backoff has elapsed, or reclaim one abandoned by a crashed worker once its lease +-- expires. Oldest first. The lease deadline comes from the DATABASE clock, the same +-- clock every ownership predicate compares it against, so app/DB skew cannot grant an +-- already-expired or over-long lease. FOR UPDATE SKIP LOCKED lets several executors +-- share the queue. Rides idx_hook_execution_lease. +UPDATE hook_execution +SET status = 'running', + lease_token = @lease_token, + lease_expires_at = clock_timestamp() + make_interval(secs => @lease_ttl_seconds::float8), + attempts = attempts + 1, + started_at = COALESCE(started_at, now()) +WHERE id = ( + SELECT id FROM hook_execution + WHERE (status = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= now())) + OR (status = 'running' AND lease_expires_at < now()) + ORDER BY created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +RETURNING *; + +-- name: GetOwnedHookExecution :one +-- Row-lock a claimed execution and assert lease OWNERSHIP before any action write. +-- Identical predicate to every terminal write below, evaluated against database clock +-- time, so a worker whose lease was reclaimed — or whose own lease elapsed — is not +-- the owner and must write nothing (§7.3: a lost lease may never write terminal state). +SELECT * FROM hook_execution +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp() +FOR UPDATE; + +-- name: AdvanceHookExecutionAction :execrows +-- Move the action cursor past a completed action, under the ownership predicate. It +-- runs in the SAME transaction as that action's target write and effect, so an action +-- can never commit without its cursor advancing. +UPDATE hook_execution +SET current_action_index = @next_action_index +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: MarkHookExecutionSucceeded :execrows +UPDATE hook_execution +SET status = 'succeeded', completed_at = now(), lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: MarkHookExecutionSkipped :execrows +-- A terminal, non-retryable outcome (§7.3): permission, unavailable target, departed +-- principal. Distinct from `failed`, which is an exhausted infrastructure retry. +UPDATE hook_execution +SET status = 'skipped', skip_reason = @skip_reason, completed_at = now(), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: MarkHookExecutionFailed :execrows +UPDATE hook_execution +SET status = 'failed', error_code = @error_code, error = @error, completed_at = now(), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: RescheduleHookExecution :execrows +-- Release the lease and re-queue for a later attempt after an infrastructure failure. +-- current_action_index is untouched, so the retry resumes at the action that failed +-- and every action already committed stays committed (§7.2 partial execution). +UPDATE hook_execution +SET status = 'queued', + next_attempt_at = now() + make_interval(secs => @backoff_seconds::int), + lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: GetHookActionEffect :one +SELECT * FROM hook_action_effect WHERE effect_key = $1; + +-- name: CreateHookActionEffect :one +-- Claim one action's idempotency anchor. ON CONFLICT DO NOTHING means a concurrent or +-- replayed attempt gets no row back and must read the existing effect instead of +-- re-running the action. +INSERT INTO hook_action_effect ( + id, effect_key, execution_id, action_index, action_type, status, resolved_input, attempts +) VALUES ($1, $2, $3, $4, $5, 'running', $6, 1) +ON CONFLICT (effect_key) DO NOTHING +RETURNING *; + +-- name: MarkHookActionEffectSucceeded :execrows +UPDATE hook_action_effect +SET status = 'succeeded', output_type = @output_type, output_id = @output_id, completed_at = now() +WHERE effect_key = $1; + +-- name: HeartbeatHookExecution :execrows +-- Extend the lease of an execution this worker still owns (§7.2). A long action must +-- not lose its lease simply because it is slow; the heartbeat keeps a live worker's +-- claim valid while an ABANDONED claim still expires on schedule. +UPDATE hook_execution +SET lease_expires_at = clock_timestamp() + make_interval(secs => @lease_ttl_seconds::float8) +WHERE id = $1 + AND status = 'running' + AND lease_token = $2 + AND lease_expires_at > clock_timestamp(); + +-- name: DeferExpiredHookExecution :execrows +-- Back off an execution whose lease elapsed while THIS worker was running it. The +-- CAS is on lease_token only — deliberately without the not-expired condition, since +-- the lease is expired by definition here — so it fires only while the row still +-- carries our token. If another worker has already reclaimed it the token differs, +-- nothing is written, and the new owner is left alone. Without this the row keeps its +-- original ordering position and the next claim selects it again immediately, +-- starving everything behind it. +UPDATE hook_execution +SET status = 'queued', + next_attempt_at = now() + make_interval(secs => @backoff_seconds::int), + lease_token = NULL, + lease_expires_at = NULL +WHERE id = $1 + AND status = 'running' + AND lease_token = $2; + +-- name: UpsertTerminalHookActionEffect :exec +-- Record the durable audit row for an action that ended terminally — skipped or +-- failed — carrying its resolved input and the reason. The success path writes its +-- effect inside the action transaction, which rolls back on failure, so without this +-- a skipped/failed action would leave no trace at all and a partial execution would +-- show only the actions that succeeded. Keyed on effect_key so a retry updates the +-- same row rather than creating a second one for the same (execution, action index). +INSERT INTO hook_action_effect ( + id, effect_key, execution_id, action_index, action_type, + status, resolved_input, attempts, error_code, error, completed_at +) VALUES ($1, $2, $3, $4, $5, @status, $6, 1, @error_code, @error, now()) +ON CONFLICT (effect_key) DO UPDATE SET + status = EXCLUDED.status, + resolved_input = EXCLUDED.resolved_input, + attempts = hook_action_effect.attempts + 1, + error_code = EXCLUDED.error_code, + error = EXCLUDED.error, + completed_at = now(); diff --git a/server/pkg/db/queries/issue.sql b/server/pkg/db/queries/issue.sql index 0d603463200..f9437f8ee6e 100644 --- a/server/pkg/db/queries/issue.sql +++ b/server/pkg/db/queries/issue.sql @@ -361,3 +361,21 @@ UPDATE issue SET first_executed_at = now() WHERE id = $1 AND first_executed_at IS NULL RETURNING id, workspace_id, creator_type, creator_id, first_executed_at; + +-- name: LockIssueRowForUpdate :one +-- Row-locks an issue and returns its full authoritative row for both a +-- concurrency-safe update and domain-event emission (MUL-4332 review points 3 & +-- 1). Callers read this INSIDE the update transaction, immediately before +-- UpdateIssue/UpdateIssueStatus. Two purposes: +-- 1. The event's `from` (status/assignee) reflects the truly-current row +-- rather than a pre-tx snapshot, so concurrent transitions serialize on +-- the lock and each records the correct edge instead of a stale `from`. +-- 2. UpdateIssue writes the nullable columns (assignee, dates, parent, +-- project, stage) as bare narg — an untouched field carries whatever the +-- caller pre-filled. Pre-filling from a pre-tx snapshot lets a concurrent +-- writer's change be silently rolled back; callers instead rebuild every +-- untouched field from THIS locked row so an unrelated update never clobbers +-- a field it did not intend to change. +SELECT * FROM issue +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE; diff --git a/server/pkg/db/queries/member.sql b/server/pkg/db/queries/member.sql index f93365d3b32..940cfcc553b 100644 --- a/server/pkg/db/queries/member.sql +++ b/server/pkg/db/queries/member.sql @@ -7,10 +7,30 @@ ORDER BY created_at ASC; SELECT * FROM member WHERE id = $1; +-- name: GetMemberInWorkspace :one +-- Workspace-scoped member existence check by member row id. Used to fail-closed +-- validate a send_inbox action target belongs to the hook's workspace +-- (MUL-4332 PR2 review point 2). +SELECT * FROM member +WHERE id = $1 AND workspace_id = $2; + -- name: GetMemberByUserAndWorkspace :one SELECT * FROM member WHERE user_id = $1 AND workspace_id = $2; +-- name: GetMemberByUserAndWorkspaceForShare :one +-- Locking membership read for authorization inside a write transaction. FOR SHARE +-- takes a shared row lock that blocks a concurrent role UPDATE (demotion) or member +-- DELETE (removal) from committing until this transaction ends, so a hook write can +-- never commit under a membership/role that was revoked mid-transaction — a plain +-- read under READ COMMITTED would miss that (MUL-4332 PR2 review round 5). Multiple +-- concurrent hook writes may still share-lock the same member without blocking each +-- other. Use ONLY inside the hook write transaction; the plain variant remains for +-- non-transactional reads. +SELECT * FROM member +WHERE user_id = $1 AND workspace_id = $2 +FOR SHARE; + -- name: CreateMember :one INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, $3) @@ -31,3 +51,12 @@ FROM member m JOIN "user" u ON u.id = m.user_id WHERE m.workspace_id = $1 ORDER BY m.created_at ASC; + +-- name: ListWorkspaceMembersByRoles :many +-- Members holding any of the given roles, for notifications that must reach the +-- people able to act on them (e.g. an automation paused because its authorization +-- principal left). Role filtering elsewhere is a per-caller authorization check; +-- this is the fan-out selector, which did not previously exist. +SELECT * FROM member +WHERE workspace_id = $1 AND role = ANY(@roles::text[]) +ORDER BY created_at ASC; diff --git a/server/pkg/db/queries/runtime.sql b/server/pkg/db/queries/runtime.sql index 6daf95f347d..9f397b6a6f3 100644 --- a/server/pkg/db/queries/runtime.sql +++ b/server/pkg/db/queries/runtime.sql @@ -226,19 +226,24 @@ WHERE status = 'online' AND last_seen_at < now() - make_interval(secs => @stale_seconds::double precision) RETURNING id, workspace_id, owner_id, daemon_id, provider; --- name: FailTasksForOfflineRuntimes :many --- Marks dispatched/running/waiting_local_directory tasks as failed when --- their runtime is offline. This cleans up orphaned tasks after a daemon --- crash or network partition. -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'runtime went offline', - failure_reason = 'runtime_offline', - wait_reason = NULL +-- name: SelectTasksForOfflineRuntimes :many +-- Selects (and row-locks) up to @max_per_tick dispatched/running/ +-- waiting_local_directory tasks whose runtime is offline — the orphans a daemon +-- crash or network partition leaves behind. This is the candidate half of the +-- MUL-4332 poison-isolated fail path (review point 2): the caller resolves each +-- task's workspace, fails only the resolvable set via FailAgentTasksByIDs and +-- emits its task.failed event in the SAME transaction, so an unresolvable poison +-- row is skipped instead of rolling back the whole batch. FOR UPDATE SKIP LOCKED +-- keeps the sweeper off rows a daemon is actively claiming; the LIMIT bounds the +-- lock hold and the rest drain on later ticks. +SELECT * FROM agent_task_queue WHERE status IN ('dispatched', 'running', 'waiting_local_directory') AND runtime_id IN ( SELECT id FROM agent_runtime WHERE status = 'offline' ) -RETURNING *; +ORDER BY created_at ASC +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; -- name: ListAgentRuntimesByOwner :many SELECT * FROM agent_runtime