From 53d758a52c1ec65b5b6a88332a58eeeff5feb87d Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Wed, 15 Jul 2026 18:05:15 +0800 Subject: [PATCH 01/15] feat(automation): transactional-outbox domain event layer (MUL-4332 PR1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR1 of the Event Hooks MVP: land the durable `domain_event` transactional outbox and converge the v1 domain write paths onto it. There is NO consumer yet — rows are written dispatch_status='pending' and nothing reads them, so this is an additive, zero-behavior-change increment. The PR3 matcher/executor will claim pending rows via the lease columns. Foundation - New `domain_event` table (MUL-4332 §4.1): fact envelope + causal chain + single-consumer outbox lease + monotonic seq. No FK / no cascade per the workspace rules; five CONCURRENTLY indexes in their own single-statement migrations; a 90-day retention sweeper (a no-op until events dispatch). - `internal/domainevent`: versioned v1 event catalog, typed payload structs, a tx-aware Write (event commits atomically with the fact) and a WriteInTx helper that wraps otherwise-bare autocommit writes. Write-path convergence (event committed in the same tx as the fact) - issue.created: HTTP create, autopilot dispatch, onboarding (both shims). - issue.status_changed + issue.assigned: single UpdateIssue, BatchUpdateIssues, GitHub merge auto-close, task-failure stuck-issue reset. - task.completed / task.failed: CompleteTask / FailTask (status CAS makes the event exactly-once on daemon-callback replay). Separate from internal/events (the in-memory Bus), which keeps serving realtime UI. Verified: all 198 migrations apply + reverse cleanly; writer atomicity (commit persists / rollback persists nothing / WriteInTx all-or-nothing) on a live DB; the full service + handler suites pass; an end-to-end test proves the outbox fires through the real HTTP handlers. Co-authored-by: multica-agent --- server/cmd/server/domain_event_retention.go | 74 ++++++ server/cmd/server/main.go | 3 + server/internal/domainevent/domainevent.go | 119 +++++++++ server/internal/domainevent/event.go | 201 +++++++++++++++ server/internal/domainevent/event_test.go | 132 ++++++++++ server/internal/domainevent/writer.go | 124 ++++++++++ server/internal/domainevent/writer_db_test.go | 204 +++++++++++++++ server/internal/featureflags/keys.go | 12 + server/internal/handler/comment.go | 45 +++- .../handler/domain_event_outbox_test.go | 131 ++++++++++ server/internal/handler/github.go | 28 ++- server/internal/handler/issue.go | 76 +++++- server/internal/handler/onboarding_shim.go | 13 + server/internal/service/autopilot.go | 7 + server/internal/service/issue.go | 8 + server/internal/service/task.go | 70 +++++- server/migrations/193_domain_event.down.sql | 5 + server/migrations/193_domain_event.up.sql | 78 ++++++ ...194_domain_event_seq_unique_index.down.sql | 1 + .../194_domain_event_seq_unique_index.up.sql | 8 + .../195_domain_event_dispatch_index.down.sql | 1 + .../195_domain_event_dispatch_index.up.sql | 5 + ...96_domain_event_correlation_index.down.sql | 1 + .../196_domain_event_correlation_index.up.sql | 5 + .../197_domain_event_type_index.down.sql | 1 + .../197_domain_event_type_index.up.sql | 5 + .../198_domain_event_subject_index.down.sql | 1 + .../198_domain_event_subject_index.up.sql | 6 + server/pkg/db/generated/domain_event.sql.go | 233 ++++++++++++++++++ server/pkg/db/generated/models.go | 25 ++ server/pkg/db/queries/domain_event.sql | 55 +++++ 31 files changed, 1646 insertions(+), 31 deletions(-) create mode 100644 server/cmd/server/domain_event_retention.go create mode 100644 server/internal/domainevent/domainevent.go create mode 100644 server/internal/domainevent/event.go create mode 100644 server/internal/domainevent/event_test.go create mode 100644 server/internal/domainevent/writer.go create mode 100644 server/internal/domainevent/writer_db_test.go create mode 100644 server/internal/handler/domain_event_outbox_test.go create mode 100644 server/migrations/193_domain_event.down.sql create mode 100644 server/migrations/193_domain_event.up.sql create mode 100644 server/migrations/194_domain_event_seq_unique_index.down.sql create mode 100644 server/migrations/194_domain_event_seq_unique_index.up.sql create mode 100644 server/migrations/195_domain_event_dispatch_index.down.sql create mode 100644 server/migrations/195_domain_event_dispatch_index.up.sql create mode 100644 server/migrations/196_domain_event_correlation_index.down.sql create mode 100644 server/migrations/196_domain_event_correlation_index.up.sql create mode 100644 server/migrations/197_domain_event_type_index.down.sql create mode 100644 server/migrations/197_domain_event_type_index.up.sql create mode 100644 server/migrations/198_domain_event_subject_index.down.sql create mode 100644 server/migrations/198_domain_event_subject_index.up.sql create mode 100644 server/pkg/db/generated/domain_event.sql.go create mode 100644 server/pkg/db/queries/domain_event.sql diff --git a/server/cmd/server/domain_event_retention.go b/server/cmd/server/domain_event_retention.go new file mode 100644 index 00000000000..c1f39aef101 --- /dev/null +++ b/server/cmd/server/domain_event_retention.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/jackc/pgx/v5/pgtype" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Domain event retention (MUL-4332 §4.1 / §9). The transactional-outbox +// domain_event table would grow without bound, so a periodic sweep reclaims +// events that are BOTH already dispatched AND older than the TTL. In PR1 there +// is no matcher, so nothing is ever marked 'dispatched' — this sweep is a +// deliberate no-op until PR3, matching the "zero behavior change" contract +// while ensuring retention lands with the table it governs. +const ( + // domainEventRetentionInterval is how often the sweep runs. Retention is + // coarse-grained, so an hourly tick is plenty. + domainEventRetentionInterval = 1 * time.Hour + // domainEventTTL is the retention window: dispatched events older than this + // are reclaimed (MUL-4332 §9 fixes this at 90 days). + domainEventTTL = 90 * 24 * time.Hour + // domainEventRetentionBatch bounds a single DELETE so a large backlog can + // never monopolize the DB; the sweep drains in batches until a short one. + domainEventRetentionBatch = 1000 +) + +// runDomainEventRetention runs the retention sweep on a ticker until ctx is +// cancelled. Registered alongside the other sweepCtx-bound workers in main so +// it stops cleanly on shutdown. +func runDomainEventRetention(ctx context.Context, queries *db.Queries) { + ticker := time.NewTicker(domainEventRetentionInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sweepDomainEvents(ctx, queries) + } + } +} + +// sweepDomainEvents deletes dispatched-and-expired events in bounded batches. +func sweepDomainEvents(ctx context.Context, queries *db.Queries) { + cutoff := pgtype.Timestamptz{Time: time.Now().Add(-domainEventTTL), Valid: true} + var total int64 + for { + deleted, err := queries.DeleteDispatchedDomainEventsBefore(ctx, db.DeleteDispatchedDomainEventsBeforeParams{ + CreatedAt: cutoff, + Limit: domainEventRetentionBatch, + }) + if err != nil { + slog.Warn("domain event retention: delete failed", "error", err) + return + } + total += deleted + // A short batch means the backlog is drained. + if deleted < domainEventRetentionBatch { + break + } + // Bail out promptly on shutdown between batches. + select { + case <-ctx.Done(): + return + default: + } + } + if total > 0 { + slog.Info("domain event retention: reclaimed dispatched events", "count", total) + } +} diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index 1d753f6b191..c17787df92f 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -386,6 +386,9 @@ func main() { // Start background sweeper to mark stale runtimes as offline. go runRuntimeSweeper(sweepCtx, queries, liveness, taskSvc, bus) + // Reclaim dispatched-and-expired domain events (MUL-4332). No-op until the + // PR3 matcher marks events dispatched. + go runDomainEventRetention(sweepCtx, queries) go heartbeatScheduler.Run(sweepCtx) go runAutopilotFailureMonitor(autopilotCtx, queries, bus, envFailureMonitorConfig()) go runDBStatsLogger(sweepCtx, pool) diff --git a/server/internal/domainevent/domainevent.go b/server/internal/domainevent/domainevent.go new file mode 100644 index 00000000000..03ff25d3d7c --- /dev/null +++ b/server/internal/domainevent/domainevent.go @@ -0,0 +1,119 @@ +// Package domainevent is the transactional-outbox event layer for the Event +// Hooks MVP (MUL-4332). It defines the versioned v1 domain event catalog and a +// tx-aware writer that persists one immutable row into the `domain_event` table +// IN THE SAME TRANSACTION as the domain fact that produced it. +// +// The contract is deliberately narrow: +// +// - A caller that already writes a domain fact inside a pgx.Tx obtains the +// tx-bound *db.Queries (via Queries.WithTx) and calls Write(ctx, qtx, evt) +// before committing. Fact and event commit atomically — a crash between +// them is impossible, which is what makes the outbox durable. +// - A caller whose write is a bare autocommit statement uses WriteInTx, which +// wraps the write + event in one transaction. +// +// PR1 has NO consumer: rows land dispatch_status='pending' and nothing reads +// them, so wiring Write into a domain path is a zero-behavior-change addition. +// The matcher/executor that claims pending rows arrives in PR3. +// +// This package is intentionally separate from internal/events (the in-memory +// events.Bus). The Bus stays best-effort, post-commit, and serves realtime UI; +// domain_event is the durable, transactional source of truth for automation. +package domainevent + +import "github.com/jackc/pgx/v5/pgtype" + +// Event type names (the `type` column). Dotted noun.verb, distinct from the +// colon-delimited events.Bus protocol names so the two namespaces never blur. +const ( + TypeIssueCreated = "issue.created" + TypeIssueStatusChanged = "issue.status_changed" + TypeIssueAssigned = "issue.assigned" + TypeCommentCreated = "comment.created" + TypeTaskCompleted = "task.completed" + TypeTaskFailed = "task.failed" + + // TypeIssueStageCompleted is a derived sensor event emitted by the PR5 + // stage frontier sensor, not by any v1 domain write. The constant is + // declared here so the catalog is complete and validators recognise it. + TypeIssueStageCompleted = "issue.stage_completed" +) + +// Subject types (the `subject_type` column): what entity the event is about. +const ( + SubjectIssue = "issue" + SubjectComment = "comment" + SubjectTask = "task" +) + +// Actor types (the `actor_type` column): who caused the event. +const ( + ActorMember = "member" + ActorAgent = "agent" + ActorSystem = "system" + ActorHook = "hook" +) + +// Dispatch statuses (the `dispatch_status` column). Only DispatchPending is +// produced in PR1; the rest are advanced by the PR3 matcher/executor. +const ( + DispatchPending = "pending" + DispatchDispatching = "dispatching" + DispatchDispatched = "dispatched" + DispatchFailed = "failed" +) + +// typeSpec pins the invariants of one event type so Write can reject a +// malformed envelope before it reaches the DB. +type typeSpec struct { + subject string + version int32 +} + +// catalog is the authoritative v1 registry. schema_version is 1 for every type +// in v1; a breaking payload change bumps the version here and the payload's +// schemaVersion() together. +var catalog = map[string]typeSpec{ + TypeIssueCreated: {subject: SubjectIssue, version: 1}, + TypeIssueStatusChanged: {subject: SubjectIssue, version: 1}, + TypeIssueAssigned: {subject: SubjectIssue, version: 1}, + TypeCommentCreated: {subject: SubjectComment, version: 1}, + TypeTaskCompleted: {subject: SubjectTask, version: 1}, + TypeTaskFailed: {subject: SubjectTask, version: 1}, + TypeIssueStageCompleted: {subject: SubjectIssue, version: 1}, +} + +var validActorTypes = map[string]bool{ + ActorMember: true, + ActorAgent: true, + ActorSystem: true, + ActorHook: true, +} + +// Actor identifies who caused an event. The ID is invalid (NULL) for a system +// actor, which has no member/agent identity. +type Actor struct { + Type string + ID pgtype.UUID +} + +// MemberActor / AgentActor / HookActor / SystemActor build an Actor from an +// identity the call site already holds. +func MemberActor(id pgtype.UUID) Actor { return Actor{Type: ActorMember, ID: id} } +func AgentActor(id pgtype.UUID) Actor { return Actor{Type: ActorAgent, ID: id} } +func HookActor(id pgtype.UUID) Actor { return Actor{Type: ActorHook, ID: id} } +func SystemActor() Actor { return Actor{Type: ActorSystem} } + +// ActorFrom builds an Actor from a raw (type, id) pair — for call sites that +// carry an existing creator_type/creator_id or author_type/author_id. An empty +// or unknown type degrades to a system actor so a mislabelled caller can never +// fabricate a member/agent identity. +func ActorFrom(actorType string, id pgtype.UUID) Actor { + if !validActorTypes[actorType] { + return SystemActor() + } + if actorType == ActorSystem { + return SystemActor() + } + return Actor{Type: actorType, ID: id} +} diff --git a/server/internal/domainevent/event.go b/server/internal/domainevent/event.go new file mode 100644 index 00000000000..0dfacafdd07 --- /dev/null +++ b/server/internal/domainevent/event.go @@ -0,0 +1,201 @@ +package domainevent + +import ( + "encoding/json" + "fmt" + + "github.com/jackc/pgx/v5/pgtype" +) + +// Event is a fully-formed, validated-on-write domain event ready to persist. +// Construct it through a typed constructor (IssueStatusChanged, CommentCreated, +// …) rather than by hand so Type / SubjectType / SchemaVersion / Payload always +// agree with the catalog. +// +// A zero CorrelationID marks a root event: Write assigns correlation_id = id and +// hop_count = 0. The Causation* / HopCount fields stay zero for every v1 domain +// write (only the PR3 executor, replaying a reaction, sets them). +type Event struct { + WorkspaceID pgtype.UUID + Type string + SchemaVersion int32 + SubjectType string + SubjectID pgtype.UUID + ActorType string + ActorID pgtype.UUID + Payload []byte + + CorrelationID pgtype.UUID + CausationExecutionID pgtype.UUID + CausationActionIndex pgtype.Int4 + HopCount int32 + + // buildErr carries a payload-marshal failure from the constructor so call + // sites stay error-free; Write surfaces it and aborts the transaction. + buildErr error +} + +// payload is implemented by every typed payload struct so a single generic +// builder can stamp the envelope fields from the payload itself. +type payload interface { + eventType() string + subjectType() string + schemaVersion() int32 +} + +func newEvent(workspaceID, subjectID pgtype.UUID, actor Actor, p payload) Event { + raw, err := json.Marshal(p) + return Event{ + WorkspaceID: workspaceID, + Type: p.eventType(), + SchemaVersion: p.schemaVersion(), + SubjectType: p.subjectType(), + SubjectID: subjectID, + ActorType: actor.Type, + ActorID: actor.ID, + Payload: raw, + buildErr: err, + } +} + +// validate rejects an envelope that disagrees with the catalog before it hits +// the DB. It is a safety net beneath the typed constructors, and the guard the +// future public REST/CLI create path will reuse. +func (e Event) validate() error { + if e.buildErr != nil { + return fmt.Errorf("domainevent: marshal payload: %w", e.buildErr) + } + spec, ok := catalog[e.Type] + if !ok { + return fmt.Errorf("domainevent: unknown event type %q", e.Type) + } + if e.SchemaVersion != spec.version { + return fmt.Errorf("domainevent: %s schema_version %d, want %d", e.Type, e.SchemaVersion, spec.version) + } + if e.SubjectType != spec.subject { + return fmt.Errorf("domainevent: %s subject_type %q, want %q", e.Type, e.SubjectType, spec.subject) + } + if !validActorTypes[e.ActorType] { + return fmt.Errorf("domainevent: invalid actor_type %q", e.ActorType) + } + if !e.WorkspaceID.Valid { + return fmt.Errorf("domainevent: %s missing workspace_id", e.Type) + } + if !e.SubjectID.Valid { + return fmt.Errorf("domainevent: %s missing subject_id", e.Type) + } + if len(e.Payload) == 0 { + return fmt.Errorf("domainevent: %s empty payload", e.Type) + } + return nil +} + +// ---- v1 payloads ---------------------------------------------------------- +// +// UUID-valued fields are JSON strings (empty + omitempty when absent). Call +// sites convert a pgtype.UUID with util.UUIDToString, which yields "" for an +// invalid/NULL value — so an absent parent/assignee is omitted, not null. + +// IssueCreatedPayload — subject is the new issue. +type IssueCreatedPayload struct { + Status string `json:"status"` + Title string `json:"title"` + Priority string `json:"priority,omitempty"` + ParentIssueID string `json:"parent_issue_id,omitempty"` + AssigneeType string `json:"assignee_type,omitempty"` + AssigneeID string `json:"assignee_id,omitempty"` + OriginType string `json:"origin_type,omitempty"` +} + +func (IssueCreatedPayload) eventType() string { return TypeIssueCreated } +func (IssueCreatedPayload) subjectType() string { return SubjectIssue } +func (IssueCreatedPayload) schemaVersion() int32 { return 1 } + +// IssueStatusChangedPayload — subject is the issue whose status moved. +type IssueStatusChangedPayload struct { + From string `json:"from"` + To string `json:"to"` +} + +func (IssueStatusChangedPayload) eventType() string { return TypeIssueStatusChanged } +func (IssueStatusChangedPayload) subjectType() string { return SubjectIssue } +func (IssueStatusChangedPayload) schemaVersion() int32 { return 1 } + +// IssueAssignedPayload — subject is the issue whose assignee changed. +type IssueAssignedPayload struct { + FromAssigneeType string `json:"from_assignee_type,omitempty"` + FromAssigneeID string `json:"from_assignee_id,omitempty"` + ToAssigneeType string `json:"to_assignee_type,omitempty"` + ToAssigneeID string `json:"to_assignee_id,omitempty"` +} + +func (IssueAssignedPayload) eventType() string { return TypeIssueAssigned } +func (IssueAssignedPayload) subjectType() string { return SubjectIssue } +func (IssueAssignedPayload) schemaVersion() int32 { return 1 } + +// CommentCreatedPayload — subject is the comment; issue_id locates its thread. +type CommentCreatedPayload struct { + IssueID string `json:"issue_id"` + AuthorType string `json:"author_type"` + AuthorID string `json:"author_id,omitempty"` + ParentID string `json:"parent_id,omitempty"` +} + +func (CommentCreatedPayload) eventType() string { return TypeCommentCreated } +func (CommentCreatedPayload) subjectType() string { return SubjectComment } +func (CommentCreatedPayload) schemaVersion() int32 { return 1 } + +// TaskCompletedPayload — subject is the task; issue_id/agent_id locate it. +type TaskCompletedPayload struct { + IssueID string `json:"issue_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` +} + +func (TaskCompletedPayload) eventType() string { return TypeTaskCompleted } +func (TaskCompletedPayload) subjectType() string { return SubjectTask } +func (TaskCompletedPayload) schemaVersion() int32 { return 1 } + +// TaskFailedPayload — subject is the task. Retryable distinguishes a terminal +// failure from one that spawned an auto-retry child. +type TaskFailedPayload struct { + IssueID string `json:"issue_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + Retryable bool `json:"retryable"` + ErrorCode string `json:"error_code,omitempty"` +} + +func (TaskFailedPayload) eventType() string { return TypeTaskFailed } +func (TaskFailedPayload) subjectType() string { return SubjectTask } +func (TaskFailedPayload) schemaVersion() int32 { return 1 } + +// ---- typed constructors --------------------------------------------------- + +// IssueCreated builds an issue.created event for the given new issue. +func IssueCreated(workspaceID, issueID pgtype.UUID, actor Actor, p IssueCreatedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// IssueStatusChanged builds an issue.status_changed event. +func IssueStatusChanged(workspaceID, issueID pgtype.UUID, actor Actor, p IssueStatusChangedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// IssueAssigned builds an issue.assigned event. +func IssueAssigned(workspaceID, issueID pgtype.UUID, actor Actor, p IssueAssignedPayload) Event { + return newEvent(workspaceID, issueID, actor, p) +} + +// CommentCreated builds a comment.created event; subjectID is the comment id. +func CommentCreated(workspaceID, commentID pgtype.UUID, actor Actor, p CommentCreatedPayload) Event { + return newEvent(workspaceID, commentID, actor, p) +} + +// TaskCompleted builds a task.completed event; subjectID is the task id. +func TaskCompleted(workspaceID, taskID pgtype.UUID, actor Actor, p TaskCompletedPayload) Event { + return newEvent(workspaceID, taskID, actor, p) +} + +// TaskFailed builds a task.failed event; subjectID is the task id. +func TaskFailed(workspaceID, taskID pgtype.UUID, actor Actor, p TaskFailedPayload) Event { + return newEvent(workspaceID, taskID, actor, p) +} diff --git a/server/internal/domainevent/event_test.go b/server/internal/domainevent/event_test.go new file mode 100644 index 00000000000..b4cb094e37c --- /dev/null +++ b/server/internal/domainevent/event_test.go @@ -0,0 +1,132 @@ +package domainevent + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +func testUUID(t *testing.T) pgtype.UUID { + t.Helper() + return pgtype.UUID{Bytes: uuid.New(), Valid: true} +} + +// Every typed constructor must produce an envelope that agrees with the catalog +// (type, subject_type, schema_version) and passes validate — this is the guard +// the future public create path reuses, so it must hold for the internal callers +// too. +func TestConstructorsProduceValidEnvelopes(t *testing.T) { + ws := testUUID(t) + subj := testUUID(t) + actor := MemberActor(testUUID(t)) + + cases := []struct { + name string + evt Event + wantType string + wantSubject string + }{ + {"issue.created", IssueCreated(ws, subj, actor, IssueCreatedPayload{Status: "todo", Title: "x"}), TypeIssueCreated, SubjectIssue}, + {"issue.status_changed", IssueStatusChanged(ws, subj, actor, IssueStatusChangedPayload{From: "todo", To: "done"}), TypeIssueStatusChanged, SubjectIssue}, + {"issue.assigned", IssueAssigned(ws, subj, actor, IssueAssignedPayload{ToAssigneeType: "agent"}), TypeIssueAssigned, SubjectIssue}, + {"comment.created", CommentCreated(ws, subj, actor, CommentCreatedPayload{IssueID: uuid.NewString(), AuthorType: "member"}), TypeCommentCreated, SubjectComment}, + {"task.completed", TaskCompleted(ws, subj, SystemActor(), TaskCompletedPayload{IssueID: uuid.NewString()}), TypeTaskCompleted, SubjectTask}, + {"task.failed", TaskFailed(ws, subj, SystemActor(), TaskFailedPayload{Retryable: true}), TypeTaskFailed, SubjectTask}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if err := tc.evt.validate(); err != nil { + t.Fatalf("validate: %v", err) + } + if tc.evt.Type != tc.wantType { + t.Errorf("Type = %q, want %q", tc.evt.Type, tc.wantType) + } + if tc.evt.SubjectType != tc.wantSubject { + t.Errorf("SubjectType = %q, want %q", tc.evt.SubjectType, tc.wantSubject) + } + if tc.evt.SchemaVersion != 1 { + t.Errorf("SchemaVersion = %d, want 1", tc.evt.SchemaVersion) + } + if !json.Valid(tc.evt.Payload) { + t.Errorf("payload is not valid JSON: %s", tc.evt.Payload) + } + // A v1 domain write is always a root event. + if tc.evt.CorrelationID.Valid || tc.evt.HopCount != 0 { + t.Errorf("expected root event (no correlation, hop 0), got correlation.Valid=%v hop=%d", tc.evt.CorrelationID.Valid, tc.evt.HopCount) + } + }) + } +} + +func TestValidateRejectsBadEnvelopes(t *testing.T) { + ws := testUUID(t) + subj := testUUID(t) + base := IssueStatusChanged(ws, subj, MemberActor(testUUID(t)), IssueStatusChangedPayload{From: "a", To: "b"}) + + mutate := func(f func(*Event)) Event { + e := base + f(&e) + return e + } + + cases := map[string]Event{ + "unknown type": mutate(func(e *Event) { e.Type = "issue.exploded" }), + "wrong schema ver": mutate(func(e *Event) { e.SchemaVersion = 2 }), + "wrong subject type": mutate(func(e *Event) { e.SubjectType = SubjectTask }), + "bad actor type": mutate(func(e *Event) { e.ActorType = "wizard" }), + "missing workspace": mutate(func(e *Event) { e.WorkspaceID = pgtype.UUID{} }), + "missing subject": mutate(func(e *Event) { e.SubjectID = pgtype.UUID{} }), + "empty payload": mutate(func(e *Event) { e.Payload = nil }), + } + for name, e := range cases { + t.Run(name, func(t *testing.T) { + if err := e.validate(); err == nil { + t.Fatalf("expected validate to reject %s", name) + } + }) + } +} + +// ActorFrom must never let an unknown or empty actor type mint a member/agent +// identity — it degrades to system so a mislabelled caller can't fabricate +// authorship (guards the accountable-actor split from MUL-4332 §8). +func TestActorFromDegradesUnknownToSystem(t *testing.T) { + id := testUUID(t) + if a := ActorFrom("member", id); a.Type != ActorMember || a.ID != id { + t.Errorf("member: got %+v", a) + } + if a := ActorFrom("", id); a.Type != ActorSystem || a.ID.Valid { + t.Errorf("empty type should degrade to system with no id, got %+v", a) + } + if a := ActorFrom("wizard", id); a.Type != ActorSystem || a.ID.Valid { + t.Errorf("unknown type should degrade to system with no id, got %+v", a) + } + if a := ActorFrom("system", id); a.Type != ActorSystem || a.ID.Valid { + t.Errorf("system actor never carries an id, got %+v", a) + } +} + +// The payload JSON shape is a wire contract the PR3 matcher's fixed-vocabulary +// predicates bind against, so pin the exact keys. +func TestPayloadJSONShape(t *testing.T) { + raw, err := json.Marshal(IssueStatusChangedPayload{From: "todo", To: "in_progress"}) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m["from"] != "todo" || m["to"] != "in_progress" { + t.Errorf("unexpected status payload: %s", raw) + } + + // omitempty must drop absent optional UUID fields rather than emit null. + raw, _ = json.Marshal(IssueCreatedPayload{Status: "todo", Title: "x"}) + if got := string(raw); got != `{"status":"todo","title":"x"}` { + t.Errorf("expected optional keys omitted, got %s", got) + } +} diff --git a/server/internal/domainevent/writer.go b/server/internal/domainevent/writer.go new file mode 100644 index 00000000000..99e933d1aaf --- /dev/null +++ b/server/internal/domainevent/writer.go @@ -0,0 +1,124 @@ +package domainevent + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// IssueCreatedFromRow builds an issue.created event from a freshly-created issue +// row, stamping its birth assignee/origin into the payload. Shared by every +// issue producer (HTTP create, autopilot dispatch, onboarding) so the event +// payload stays identical regardless of which path created the issue. +func IssueCreatedFromRow(issue db.Issue) Event { + return IssueCreated(issue.WorkspaceID, issue.ID, + ActorFrom(issue.CreatorType, issue.CreatorID), + IssueCreatedPayload{ + Status: issue.Status, + Title: issue.Title, + Priority: issue.Priority, + ParentIssueID: util.UUIDToString(issue.ParentIssueID), + AssigneeType: issue.AssigneeType.String, + AssigneeID: util.UUIDToString(issue.AssigneeID), + OriginType: issue.OriginType.String, + }) +} + +// Creator is the single DB method Write needs. *db.Queries satisfies it, so a +// caller passes the tx-bound handle it already holds (base.WithTx(tx)). Keeping +// it an interface (not *db.Queries) lets tests substitute a fake. +type Creator interface { + CreateDomainEvent(ctx context.Context, arg db.CreateDomainEventParams) (db.DomainEvent, error) +} + +// txBeginner is the subset of *pgxpool.Pool that WriteInTx needs. +type txBeginner interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + +// Write validates evt and inserts it as a pending outbox row using q, which MUST +// be bound to the caller's transaction (Queries.WithTx) so the event commits +// atomically with the domain fact. It returns the persisted row (seq/created_at +// populated by the DB). +// +// For a root event (CorrelationID unset — every v1 domain write) Write assigns a +// fresh id and sets correlation_id = id, hop_count = 0. +func Write(ctx context.Context, q Creator, evt Event) (db.DomainEvent, error) { + if err := evt.validate(); err != nil { + return db.DomainEvent{}, err + } + + id := pgUUID(uuid.New()) + correlation := evt.CorrelationID + if !correlation.Valid { + // Root event: it is its own correlation head. + correlation = id + } + + row, err := q.CreateDomainEvent(ctx, db.CreateDomainEventParams{ + ID: id, + WorkspaceID: evt.WorkspaceID, + Type: evt.Type, + SchemaVersion: evt.SchemaVersion, + SubjectType: evt.SubjectType, + SubjectID: evt.SubjectID, + ActorType: evt.ActorType, + ActorID: evt.ActorID, + Payload: evt.Payload, + CorrelationID: correlation, + CausationExecutionID: evt.CausationExecutionID, + CausationActionIndex: evt.CausationActionIndex, + HopCount: evt.HopCount, + }) + if err != nil { + return db.DomainEvent{}, fmt.Errorf("domainevent: insert %s: %w", evt.Type, err) + } + return row, nil +} + +// WriteInTx wraps a domain write that is otherwise a bare autocommit statement. +// It opens a transaction, hands fn a tx-bound *db.Queries to perform the write, +// then persists the events fn returns — all in one commit, so the fact and its +// events land atomically or not at all. +// +// err := domainevent.WriteInTx(ctx, pool, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { +// row, err := qtx.UpdateIssueStatus(ctx, params) +// if err != nil { return nil, err } +// return []domainevent.Event{domainevent.IssueStatusChanged(ws, row.ID, actor, ...)}, nil +// }) +// +// base is the pool-bound *db.Queries; WriteInTx rebinds it to the new tx. If fn +// returns no events (e.g. the write turned out to be a no-op), the transaction +// still commits the fn's own writes. +func WriteInTx(ctx context.Context, tb txBeginner, base *db.Queries, fn func(qtx *db.Queries) ([]Event, error)) error { + tx, err := tb.Begin(ctx) + if err != nil { + return fmt.Errorf("domainevent: begin tx: %w", err) + } + defer tx.Rollback(ctx) + + qtx := base.WithTx(tx) + events, err := fn(qtx) + if err != nil { + return err + } + for _, evt := range events { + if _, err := Write(ctx, qtx, evt); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("domainevent: commit tx: %w", err) + } + return nil +} + +func pgUUID(u uuid.UUID) pgtype.UUID { + return pgtype.UUID{Bytes: u, Valid: true} +} diff --git a/server/internal/domainevent/writer_db_test.go b/server/internal/domainevent/writer_db_test.go new file mode 100644 index 00000000000..d8c72c26743 --- /dev/null +++ b/server/internal/domainevent/writer_db_test.go @@ -0,0 +1,204 @@ +package domainevent + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// newDomainEventTestPool connects to the shared, already-migrated test DB. Like +// every other DB-backed test in this repo it SKIPS (never fails) when no +// database is reachable — the schema is migrated out of band by `make test`/CI. +func newDomainEventTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + dbURL = "postgres://multica:multica@localhost:5432/multica?sslmode=disable" + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + pool, err := pgxpool.New(ctx, dbURL) + if err != nil { + t.Skipf("database unavailable: %v", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + t.Skipf("database unreachable: %v", err) + } + // Confirm the migration is applied; a DB pinned to an older schema should + // skip, not fail with a missing-relation error. + if _, err := pool.Exec(ctx, "SELECT 1 FROM domain_event LIMIT 0"); err != nil { + pool.Close() + t.Skipf("domain_event table missing (migrate up first): %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func countEventsForWorkspace(t *testing.T, pool *pgxpool.Pool, ws pgtype.UUID) int { + t.Helper() + var n int + if err := pool.QueryRow(context.Background(), + `SELECT count(*) FROM domain_event WHERE workspace_id = $1`, ws).Scan(&n); err != nil { + t.Fatalf("count: %v", err) + } + return n +} + +// cleanupWorkspaceEvents deletes rows with context.Background so it survives a +// cancelled test context. +func cleanupWorkspaceEvents(pool *pgxpool.Pool, ws pgtype.UUID) { + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE workspace_id = $1`, ws) +} + +func standInFactEvent(ws pgtype.UUID) Event { + return IssueCreated(ws, pgUUID(uuid.New()), SystemActor(), IssueCreatedPayload{Status: "todo", Title: "stand-in fact"}) +} + +// A committed Write must persist exactly one row and stamp the root-event +// invariants: dispatch_status='pending', hop_count=0, correlation_id=id, a +// monotonic seq, and the exact payload. +func TestWriteCommitPersistsRootEvent(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + ws := pgUUID(uuid.New()) + subj := pgUUID(uuid.New()) + actor := MemberActor(pgUUID(uuid.New())) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + + evt := IssueStatusChanged(ws, subj, actor, IssueStatusChangedPayload{From: "todo", To: "done"}) + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + row, err := Write(ctx, queries.WithTx(tx), evt) + if err != nil { + tx.Rollback(ctx) + t.Fatalf("write: %v", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit: %v", err) + } + + if got := countEventsForWorkspace(t, pool, ws); got != 1 { + t.Fatalf("expected exactly 1 event, got %d", got) + } + if row.DispatchStatus != DispatchPending { + t.Errorf("dispatch_status = %q, want %q", row.DispatchStatus, DispatchPending) + } + if row.HopCount != 0 { + t.Errorf("hop_count = %d, want 0", row.HopCount) + } + if row.CorrelationID != row.ID { + t.Errorf("root event correlation_id (%v) must equal id (%v)", row.CorrelationID, row.ID) + } + if row.Seq <= 0 { + t.Errorf("seq should be a positive monotonic value, got %d", row.Seq) + } + if row.Type != TypeIssueStatusChanged { + t.Errorf("type = %q", row.Type) + } +} + +// The outbox durability invariant (MUL-4332 §1 kill test #1): a crash after the +// domain write but before commit must leave NO event — the write and the event +// share one transaction, so an uncommitted event simply does not exist. +func TestWriteRollbackPersistsNothing(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := Write(ctx, queries.WithTx(tx), standInFactEvent(ws)); err != nil { + tx.Rollback(ctx) + t.Fatalf("write: %v", err) + } + // Simulate the process dying before commit. + if err := tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("rolled-back event must not persist, found %d rows", got) + } +} + +// WriteInTx must be all-or-nothing: if the caller's domain write or its own +// event insert fails, the fact written inside fn rolls back too. +func TestWriteInTxAtomicity(t *testing.T) { + pool := newDomainEventTestPool(t) + ctx := context.Background() + queries := db.New(pool) + + t.Run("commit persists fact and event", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + // Stand-in "domain write" inside the tx. + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + return []Event{standInFactEvent(ws)}, nil + }) + if err != nil { + t.Fatalf("WriteInTx: %v", err) + } + if got := countEventsForWorkspace(t, pool, ws); got != 2 { + t.Fatalf("expected fact + event = 2 rows, got %d", got) + } + }) + + t.Run("fn error rolls back the fact", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + boom := errors.New("domain write failed") + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + return nil, boom + }) + if !errors.Is(err, boom) { + t.Fatalf("expected boom, got %v", err) + } + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("fn error must roll back the fact, found %d rows", got) + } + }) + + t.Run("invalid event rolls back the fact", func(t *testing.T) { + ws := pgUUID(uuid.New()) + t.Cleanup(func() { cleanupWorkspaceEvents(pool, ws) }) + err := WriteInTx(ctx, pool, queries, func(qtx *db.Queries) ([]Event, error) { + if _, err := Write(ctx, qtx, standInFactEvent(ws)); err != nil { + return nil, err + } + bad := standInFactEvent(ws) + bad.Type = "issue.exploded" // fails validate in Write + return []Event{bad}, nil + }) + if err == nil { + t.Fatal("expected invalid event to fail WriteInTx") + } + if got := countEventsForWorkspace(t, pool, ws); got != 0 { + t.Fatalf("invalid event must roll back the fact, found %d rows", got) + } + }) +} diff --git a/server/internal/featureflags/keys.go b/server/internal/featureflags/keys.go index 5d1f78e0848..c99c1d2a7e4 100644 --- a/server/internal/featureflags/keys.go +++ b/server/internal/featureflags/keys.go @@ -23,6 +23,11 @@ const ( // key as enabled so installed v0.4.0 desktop clients, which still gate the // switch on this config decision, receive the permanently enabled behavior. agentSkillTogglesCompat = "agents_skill_toggles" + // EventHooks gates the Event Hooks engine (MUL-4332). PR1 only lands the + // transactional-outbox event layer, which is always-on and consumer-less; + // this flag stays off until the PR3 matcher/executor ships so no reaction + // ever fires from a partially-built engine. Server-only, default off. + EventHooks = "automation_event_hooks" ) var frontendPublicFlags = []string{ @@ -43,6 +48,13 @@ func ResourceLabelsEnabled(ctx context.Context, flags *featureflag.Service) bool return flags.IsEnabled(ctx, ResourceLabels, false) } +// EventHooksEnabled reports whether the Event Hooks engine may run reactions. +// PR1 does not consult it (the outbox writer is always-on); it exists so the +// PR3 matcher/executor can gate execution behind a default-off switch. +func EventHooksEnabled(ctx context.Context, flags *featureflag.Service) bool { + return flags.IsEnabled(ctx, EventHooks, false) +} + func EvaluateFrontendPublicFlags(ctx context.Context, flags *featureflag.Service) map[string]bool { out := make(map[string]bool, len(frontendPublicFlags)+1) for _, key := range frontendPublicFlags { diff --git a/server/internal/handler/comment.go b/server/internal/handler/comment.go index 5103460b770..d575e5566d3 100644 --- a/server/internal/handler/comment.go +++ b/server/internal/handler/comment.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/logger" "github.com/multica-ai/multica/server/internal/service" "github.com/multica-ai/multica/server/internal/util" @@ -1356,19 +1357,37 @@ func (h *Handler) CreateComment(w http.ResponseWriter, r *http.Request) { } } - comment, err := h.Queries.CreateComment(r.Context(), db.CreateCommentParams{ - IssueID: issue.ID, - WorkspaceID: issue.WorkspaceID, - AuthorType: authorType, - AuthorID: parseUUID(authorID), - Content: req.Content, - Type: req.Type, - ParentID: parentID, - SourceTaskID: sourceTaskID, - }) - if err != nil { - slog.Warn("create comment failed", append(logger.RequestAttrs(r), "error", err, "issue_id", issueID)...) - writeError(w, http.StatusInternalServerError, "failed to create comment: "+err.Error()) + // Transactional outbox (MUL-4332): the comment and its comment.created event + // commit in one transaction. subject is the comment; the payload locates its + // issue/thread so a hook can react without a second lookup. + commentActorUUID, _ := util.ParseUUID(authorID) + commentActor := domainevent.ActorFrom(authorType, commentActorUUID) + var comment db.Comment + if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, err := qtx.CreateComment(r.Context(), db.CreateCommentParams{ + IssueID: issue.ID, + WorkspaceID: issue.WorkspaceID, + AuthorType: authorType, + AuthorID: parseUUID(authorID), + Content: req.Content, + Type: req.Type, + ParentID: parentID, + SourceTaskID: sourceTaskID, + }) + if err != nil { + return nil, err + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, commentActor, + domainevent.CommentCreatedPayload{ + IssueID: uuidToString(created.IssueID), + AuthorType: authorType, + AuthorID: authorID, + ParentID: uuidToString(parentID), + })}, nil + }); writeErr != nil { + slog.Warn("create comment failed", append(logger.RequestAttrs(r), "error", writeErr, "issue_id", issueID)...) + writeError(w, http.StatusInternalServerError, "failed to create comment: "+writeErr.Error()) return } diff --git a/server/internal/handler/domain_event_outbox_test.go b/server/internal/handler/domain_event_outbox_test.go new file mode 100644 index 00000000000..1a2ad91343c --- /dev/null +++ b/server/internal/handler/domain_event_outbox_test.go @@ -0,0 +1,131 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/multica-ai/multica/server/internal/domainevent" +) + +type outboxRow struct { + Type string + Payload []byte + ID string + CorrelationID string + DispatchStatus string + HopCount int32 +} + +// eventsForSubject returns every domain_event about a specific subject, ordered +// by seq. Scoping by subject_id (not the shared test workspace) keeps the read +// isolated from other tests running against the same DB. +func eventsForSubject(t *testing.T, subjectType, subjectID string) []outboxRow { + t.Helper() + rows, err := testPool.Query(context.Background(), + `SELECT type, payload, id::text, correlation_id::text, dispatch_status, hop_count + FROM domain_event + WHERE subject_type = $1 AND subject_id = $2 + ORDER BY seq`, subjectType, subjectID) + if err != nil { + t.Fatalf("query domain_event: %v", err) + } + defer rows.Close() + var out []outboxRow + for rows.Next() { + var r outboxRow + if err := rows.Scan(&r.Type, &r.Payload, &r.ID, &r.CorrelationID, &r.DispatchStatus, &r.HopCount); err != nil { + t.Fatalf("scan: %v", err) + } + out = append(out, r) + } + return out +} + +func payloadField(t *testing.T, raw []byte, key string) string { + t.Helper() + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + s, _ := m[key].(string) + return s +} + +// End-to-end: driving the real HTTP handlers must persist the transactional +// outbox events (MUL-4332). Proves issue.created / issue.status_changed / +// issue.assigned are written atomically by the create + update paths, with the +// root-event invariants (pending, hop 0, correlation = id) intact. +func TestOutboxEmittedByIssueHandlers(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + issueID := createTestIssue(t, "outbox e2e "+t.Name(), "todo", "none") + t.Cleanup(func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // 1) create → exactly one issue.created, a root event. + created := eventsForSubject(t, domainevent.SubjectIssue, issueID) + if len(created) != 1 { + t.Fatalf("expected 1 event after create, got %d (%+v)", len(created), created) + } + ev := created[0] + if ev.Type != domainevent.TypeIssueCreated { + t.Errorf("type = %q, want %q", ev.Type, domainevent.TypeIssueCreated) + } + if ev.DispatchStatus != domainevent.DispatchPending { + t.Errorf("dispatch_status = %q, want pending", ev.DispatchStatus) + } + if ev.HopCount != 0 { + t.Errorf("hop_count = %d, want 0", ev.HopCount) + } + if ev.CorrelationID != ev.ID { + t.Errorf("root correlation_id (%s) must equal id (%s)", ev.CorrelationID, ev.ID) + } + if got := payloadField(t, ev.Payload, "status"); got != "todo" { + t.Errorf("issue.created payload status = %q, want todo", got) + } + + // 2) update status + assignee in one call → status_changed + assigned. + uw := httptest.NewRecorder() + ureq := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{ + "status": "in_progress", + "assignee_type": "member", + "assignee_id": testUserID, + }) + ureq = withURLParam(ureq, "id", issueID) + testHandler.UpdateIssue(uw, ureq) + if uw.Code != http.StatusOK { + t.Fatalf("UpdateIssue: expected 200, got %d: %s", uw.Code, uw.Body.String()) + } + + all := eventsForSubject(t, domainevent.SubjectIssue, issueID) + var sawStatus, sawAssigned bool + for _, e := range all { + switch e.Type { + case domainevent.TypeIssueStatusChanged: + sawStatus = true + if from := payloadField(t, e.Payload, "from"); from != "todo" { + t.Errorf("status_changed from = %q, want todo", from) + } + if to := payloadField(t, e.Payload, "to"); to != "in_progress" { + t.Errorf("status_changed to = %q, want in_progress", to) + } + case domainevent.TypeIssueAssigned: + sawAssigned = true + if to := payloadField(t, e.Payload, "to_assignee_id"); to != testUserID { + t.Errorf("assigned to_assignee_id = %q, want %q", to, testUserID) + } + } + } + if !sawStatus { + t.Errorf("expected an issue.status_changed event, got %+v", all) + } + if !sawAssigned { + t.Errorf("expected an issue.assigned event, got %+v", all) + } +} diff --git a/server/internal/handler/github.go b/server/internal/handler/github.go index 1f45430af77..9fd20f05122 100644 --- a/server/internal/handler/github.go +++ b/server/internal/handler/github.go @@ -23,6 +23,7 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/middleware" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" @@ -1361,12 +1362,27 @@ func (h *Handler) lookupIssueByIdentifier(ctx context.Context, workspaceID pgtyp } func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, workspaceID string) { - updated, err := h.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ - ID: issue.ID, - Status: "done", - WorkspaceID: issue.WorkspaceID, - }) - if err != nil { + // Transactional outbox (MUL-4332): a merged PR closing an issue is one of the + // most common status transitions to `done`, so emit issue.status_changed + // atomically with the status flip — but only on a real transition (an + // already-done issue produces no event). + var updated db.Issue + if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + u, err := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ + ID: issue.ID, + Status: "done", + WorkspaceID: issue.WorkspaceID, + }) + if err != nil { + return nil, err + } + updated = u + if issue.Status == u.Status { + return nil, nil + } + return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), + domainevent.IssueStatusChangedPayload{From: issue.Status, To: u.Status})}, nil + }); err != nil { slog.Warn("github: advance issue to done failed", "err", err) return } diff --git a/server/internal/handler/issue.go b/server/internal/handler/issue.go index 13849c9d69e..676e1d263c7 100644 --- a/server/internal/handler/issue.go +++ b/server/internal/handler/issue.go @@ -18,6 +18,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/logger" "github.com/multica-ai/multica/server/internal/middleware" @@ -2687,7 +2688,41 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { return } - issue, err := h.Queries.UpdateIssue(r.Context(), params) + // Resolve the acting identity once, up here, so the transactional-outbox + // events below and the realtime publish further down share one actor. + actorType, actorID := h.resolveActor(r, userID, workspaceID) + actorUUID, _ := util.ParseUUID(actorID) + eventActor := domainevent.ActorFrom(actorType, actorUUID) + + // Transactional outbox (MUL-4332): commit the issue update and any derived + // status_changed / assigned event in one transaction, so a crash can never + // separate the fact from its event. The events are computed by diffing + // prevIssue against the freshly-updated row. + var issue db.Issue + err = domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + updated, err := qtx.UpdateIssue(r.Context(), params) + if err != nil { + return nil, err + } + issue = updated + + var events []domainevent.Event + if req.Status != nil && prevIssue.Status != updated.Status { + events = append(events, domainevent.IssueStatusChanged(updated.WorkspaceID, updated.ID, eventActor, + domainevent.IssueStatusChangedPayload{From: prevIssue.Status, To: updated.Status})) + } + if (req.AssigneeType != nil || req.AssigneeID != nil) && + (prevIssue.AssigneeType.String != updated.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(updated.AssigneeID)) { + events = append(events, domainevent.IssueAssigned(updated.WorkspaceID, updated.ID, eventActor, + domainevent.IssueAssignedPayload{ + FromAssigneeType: prevIssue.AssigneeType.String, + FromAssigneeID: uuidToString(prevIssue.AssigneeID), + ToAssigneeType: updated.AssigneeType.String, + ToAssigneeID: uuidToString(updated.AssigneeID), + })) + } + return events, nil + }) if err != nil { slog.Warn("update issue failed", append(logger.RequestAttrs(r), "error", err, "issue_id", id, "workspace_id", workspaceID)...) writeError(w, http.StatusInternalServerError, "failed to update issue: "+err.Error()) @@ -2720,9 +2755,7 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { dueDateChanged := prevDueDate != resp.DueDate && (prevDueDate == nil) != (resp.DueDate == nil) || (prevDueDate != nil && resp.DueDate != nil && *prevDueDate != *resp.DueDate) - // Determine actor identity: agent (via X-Agent-ID header) or member. - actorType, actorID := h.resolveActor(r, userID, workspaceID) - + // actorType / actorID were resolved above (shared with the domain events). h.publish(protocol.EventIssueUpdated, workspaceID, actorType, actorID, map[string]any{ "issue": resp, "assignee_changed": assigneeChanged, @@ -3230,15 +3263,42 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { } } - issue, err := h.Queries.UpdateIssue(r.Context(), params) - if err != nil { - slog.Warn("batch update issue failed", "issue_id", issueID, "error", err) + actorType, actorID := h.resolveActor(r, userID, workspaceID) + batchActorUUID, _ := util.ParseUUID(actorID) + batchEventActor := domainevent.ActorFrom(actorType, batchActorUUID) + + // Transactional outbox (MUL-4332): each row's update and its derived + // status_changed / assigned events commit atomically, per issue. + var issue db.Issue + if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + updatedIssue, err := qtx.UpdateIssue(r.Context(), params) + if err != nil { + return nil, err + } + issue = updatedIssue + var events []domainevent.Event + if req.Updates.Status != nil && prevIssue.Status != updatedIssue.Status { + events = append(events, domainevent.IssueStatusChanged(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, + domainevent.IssueStatusChangedPayload{From: prevIssue.Status, To: updatedIssue.Status})) + } + if (req.Updates.AssigneeType != nil || req.Updates.AssigneeID != nil) && + (prevIssue.AssigneeType.String != updatedIssue.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(updatedIssue.AssigneeID)) { + events = append(events, domainevent.IssueAssigned(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, + domainevent.IssueAssignedPayload{ + FromAssigneeType: prevIssue.AssigneeType.String, + FromAssigneeID: uuidToString(prevIssue.AssigneeID), + ToAssigneeType: updatedIssue.AssigneeType.String, + ToAssigneeID: uuidToString(updatedIssue.AssigneeID), + })) + } + return events, nil + }); writeErr != nil { + slog.Warn("batch update issue failed", "issue_id", issueID, "error", writeErr) continue } prefix := h.getIssuePrefix(r.Context(), issue.WorkspaceID) resp := issueToResponse(issue, prefix) - actorType, actorID := h.resolveActor(r, userID, workspaceID) assigneeChanged := (req.Updates.AssigneeType != nil || req.Updates.AssigneeID != nil) && (prevIssue.AssigneeType.String != issue.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(issue.AssigneeID)) diff --git a/server/internal/handler/onboarding_shim.go b/server/internal/handler/onboarding_shim.go index f460cc6b57b..0362b190222 100644 --- a/server/internal/handler/onboarding_shim.go +++ b/server/internal/handler/onboarding_shim.go @@ -31,6 +31,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/logger" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" @@ -278,6 +279,12 @@ func (h *Handler) BootstrapOnboardingRuntime(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") return } + // Transactional outbox (MUL-4332): onboarding issue.created. + if _, err := domainevent.Write(r.Context(), qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + slog.Warn("bootstrap onboarding (shim): write issue.created event failed", append(logger.RequestAttrs(r), "error", err)...) + writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") + return + } issueCreated = true } @@ -436,6 +443,12 @@ func (h *Handler) BootstrapOnboardingNoRuntime(w http.ResponseWriter, r *http.Re writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") return } + // Transactional outbox (MUL-4332): onboarding issue.created. + if _, err := domainevent.Write(r.Context(), qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + slog.Warn("bootstrap no-runtime onboarding (shim): write issue.created event failed", append(logger.RequestAttrs(r), "error", err)...) + writeError(w, http.StatusInternalServerError, "failed to create onboarding issue") + return + } issueCreated = true } diff --git a/server/internal/service/autopilot.go b/server/internal/service/autopilot.go index 03831e0e168..c3d0373a7c1 100644 --- a/server/internal/service/autopilot.go +++ b/server/internal/service/autopilot.go @@ -16,6 +16,7 @@ import ( "github.com/multica-ai/multica/server/internal/analytics" "github.com/multica-ai/multica/server/internal/attribution" "github.com/multica-ai/multica/server/internal/dispatch" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/issueposition" @@ -676,6 +677,12 @@ func (s *AutopilotService) dispatchCreateIssue(ctx context.Context, ap db.Autopi } *run = updatedRun + // Transactional outbox (MUL-4332): emit issue.created atomically with the + // autopilot-dispatched issue insert, same as the HTTP create path. + if _, err := domainevent.Write(ctx, qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + return fmt.Errorf("write issue.created event: %w", err) + } + if err := tx.Commit(ctx); err != nil { return fmt.Errorf("commit tx: %w", err) } diff --git a/server/internal/service/issue.go b/server/internal/service/issue.go index 124d8c5ab93..6b8257f3e40 100644 --- a/server/internal/service/issue.go +++ b/server/internal/service/issue.go @@ -9,6 +9,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/issueguard" "github.com/multica-ai/multica/server/internal/issueposition" @@ -315,6 +316,13 @@ func (s *IssueService) Create(ctx context.Context, p IssueCreateParams, opts Iss } } + // Transactional outbox (MUL-4332): the issue.created fact and its domain + // event commit atomically. The payload carries the birth assignee so a hook + // can react to assignment-at-create without a separate issue.assigned event. + if _, err := domainevent.Write(ctx, qtx, domainevent.IssueCreatedFromRow(issue)); err != nil { + return IssueCreateResult{}, fmt.Errorf("write issue.created event: %w", err) + } + if err := tx.Commit(ctx); err != nil { return IssueCreateResult{}, fmt.Errorf("commit: %w", err) } diff --git a/server/internal/service/task.go b/server/internal/service/task.go index 377856b185d..465d633f708 100644 --- a/server/internal/service/task.go +++ b/server/internal/service/task.go @@ -16,6 +16,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/analytics" "github.com/multica-ai/multica/server/internal/attribution" + "github.com/multica-ai/multica/server/internal/domainevent" "github.com/multica-ai/multica/server/internal/events" "github.com/multica-ai/multica/server/internal/featureflags" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" @@ -2625,6 +2626,20 @@ func (s *TaskService) CompleteTask(ctx context.Context, taskID pgtype.UUID, resu } chatAssistantMsg = msg } + + // Transactional outbox (MUL-4332): emit task.completed atomically with + // the status flip. The status CAS above makes this exactly-once — a + // replayed callback finds the task already terminal and never re-emits. + if wsID := s.taskWorkspaceID(ctx, qtx, t.AgentID); wsID.Valid { + evt := domainevent.TaskCompleted(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskCompletedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.completed event: %w", err) + } + } return nil }); err != nil { // When parallel agents race, a task may already be completed, @@ -2953,6 +2968,22 @@ func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, } retried = &child } + + // Transactional outbox (MUL-4332): emit task.failed atomically with the + // fail (and any retry child). retryable reflects whether this failure + // spawned an auto-retry; the status CAS makes it exactly-once. + if wsID := s.taskWorkspaceID(ctx, qtx, t.AgentID); wsID.Valid { + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskFailedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + Retryable: wantRetry, + ErrorCode: failureReason, + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.failed event: %w", err) + } + } return nil }); err != nil { if existing, lookupErr := s.Queries.GetAgentTask(ctx, taskID); lookupErr == nil { @@ -3431,10 +3462,21 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas "error", checkErr, ) } else if !hasActive { - updatedIssue, updateErr := s.Queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ - ID: t.IssueID, - Status: "todo", - WorkspaceID: issue.WorkspaceID, + var updatedIssue db.Issue + // Transactional outbox (MUL-4332): the stuck-issue reset bypasses the + // HTTP UpdateIssue path, so emit issue.status_changed atomically here. + updateErr := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + u, uErr := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ + ID: t.IssueID, + Status: "todo", + WorkspaceID: issue.WorkspaceID, + }) + if uErr != nil { + return nil, uErr + } + updatedIssue = u + return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), + domainevent.IssueStatusChangedPayload{From: issue.Status, To: u.Status})}, nil }) if updateErr != nil { slog.Warn("handle failed tasks: reset stuck issue failed", @@ -3485,6 +3527,26 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas // runInTx executes fn inside a single DB transaction. If TxStarter is nil // (e.g. some tests construct TaskService directly), fn runs against the // regular Queries handle without transactional guarantees. +// taskWorkspaceID resolves the workspace of a task via its agent, to stamp the +// workspace_id of a task domain event (agent_task_queue itself carries no +// workspace column). It reads on the passed qtx so the lookup stays in the +// caller's transaction. Best-effort: an unresolvable agent (already deleted) +// returns an invalid UUID and the caller skips the event rather than wedging +// the task's terminal callback — a missing agent means the workspace is +// genuinely unknowable, and the lost event is bounded to that pathological case. +func (s *TaskService) taskWorkspaceID(ctx context.Context, qtx *db.Queries, agentID pgtype.UUID) pgtype.UUID { + if !agentID.Valid { + return pgtype.UUID{} + } + agent, err := qtx.GetAgent(ctx, agentID) + if err != nil { + slog.Warn("domain event: could not resolve task workspace via agent", + "agent_id", util.UUIDToString(agentID), "error", err) + return pgtype.UUID{} + } + return agent.WorkspaceID +} + func (s *TaskService) runInTx(ctx context.Context, fn func(*db.Queries) error) error { if s.TxStarter == nil { return fn(s.Queries) diff --git a/server/migrations/193_domain_event.down.sql b/server/migrations/193_domain_event.down.sql new file mode 100644 index 00000000000..e1d89df7ed7 --- /dev/null +++ b/server/migrations/193_domain_event.down.sql @@ -0,0 +1,5 @@ +-- Reverses 193_domain_event.up.sql. Dropping the table also drops the +-- sequence it OWNS; the explicit DROP SEQUENCE is a defensive no-op in case +-- the ownership link is ever severed. +DROP TABLE IF EXISTS domain_event; +DROP SEQUENCE IF EXISTS domain_event_seq; diff --git a/server/migrations/193_domain_event.up.sql b/server/migrations/193_domain_event.up.sql new file mode 100644 index 00000000000..e8cc9e0c0fd --- /dev/null +++ b/server/migrations/193_domain_event.up.sql @@ -0,0 +1,78 @@ +-- Event Hooks MVP — PR1: the transactional-outbox domain event log (MUL-4332 §4.1). +-- +-- `domain_event` is the persisted source of truth for the future hooks engine. +-- Every domain command that commits a fact (issue created / status changed / +-- assigned, comment created, task completed / failed) will ALSO insert one row +-- here IN THE SAME TRANSACTION, so the fact and its event commit atomically — +-- the classic transactional outbox. A crash after the domain write but before +-- any downstream reaction can therefore never lose the event. +-- +-- PR1 ships the table + the write-path convergence only. There is NO consumer +-- yet: rows land with dispatch_status='pending' and nothing reads them, so this +-- is a zero-behavior-change, additive migration. The matcher/executor that +-- claims pending rows via the lease columns arrives in PR3. +-- +-- The in-memory `events.Bus` (internal/events) is SEPARATE and unchanged — it +-- keeps serving realtime UI fanout. This table is the durable, transactional +-- fact log; the Bus is best-effort post-commit push. They do not replace each +-- other. +-- +-- Workspace DB rules (CLAUDE.md + MUL-4332 §4): NO foreign key, NO cascade — +-- every UUID association is validated in the application layer. Secondary and +-- unique indexes are added in their own single-statement CONCURRENTLY +-- migrations (194–198), never inline, so index builds never take an ACCESS +-- EXCLUSIVE lock during deploy. + +-- Monotonic dispatch / drain boundary. `seq` orders events for stable scanning +-- and the future issue-scope drain boundary; it is NOT a promise of global +-- business ordering across all actions (MUL-4332 §4.1). A sequence (not a +-- serial/identity column) so the ownership is explicit and the down migration +-- can drop it cleanly. +CREATE SEQUENCE IF NOT EXISTS domain_event_seq; + +CREATE TABLE IF NOT EXISTS domain_event ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + seq BIGINT NOT NULL DEFAULT nextval('domain_event_seq'), + + -- Fact envelope. + workspace_id UUID NOT NULL, + type TEXT NOT NULL, -- e.g. issue.status_changed + schema_version INT NOT NULL, -- per-type payload schema version + subject_type TEXT NOT NULL, -- issue | comment | task + subject_id UUID NOT NULL, + actor_type TEXT NOT NULL, -- member | agent | system | hook + actor_id UUID, -- NULL for system actors + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- Causal chain. A human root event has correlation_id = id and hop_count = 0; + -- events produced by a future hook action inherit the correlation, record the + -- producing execution/action, and increment hop_count (guardrail in PR3). + correlation_id UUID NOT NULL, + causation_execution_id UUID, + causation_action_index INT, + hop_count INT NOT NULL DEFAULT 0, + + -- Single-consumer outbox lease. Unused in PR1 (no matcher); rows stay + -- 'pending'. PR3's matcher claims via lease_token/lease_expires_at and + -- advances dispatch_status. + dispatch_status TEXT NOT NULL DEFAULT 'pending' + CHECK (dispatch_status IN ('pending', 'dispatching', 'dispatched', 'failed')), + attempts INT NOT NULL DEFAULT 0, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + dispatched_at TIMESTAMPTZ, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Tie the sequence lifecycle to the column so a table drop reclaims it. +ALTER SEQUENCE domain_event_seq OWNED BY domain_event.seq; + +-- Application-layer integrity only: proactively drop any *_fkey a tool might +-- add, matching the workspace no-FK / no-cascade rule (see 186_autopilot_rule_version). +ALTER TABLE domain_event + DROP CONSTRAINT IF EXISTS domain_event_workspace_id_fkey; + +COMMENT ON TABLE domain_event IS + 'Transactional-outbox domain event log (MUL-4332 §4.1). One row per committed domain fact, written in the same tx as the fact. Source of truth for the hooks engine; no FK, no cascade, app-layer integrity. dispatch_* columns are the single-consumer outbox lease consumed by the PR3 matcher.'; diff --git a/server/migrations/194_domain_event_seq_unique_index.down.sql b/server/migrations/194_domain_event_seq_unique_index.down.sql new file mode 100644 index 00000000000..9bf30a64858 --- /dev/null +++ b/server/migrations/194_domain_event_seq_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_seq; diff --git a/server/migrations/194_domain_event_seq_unique_index.up.sql b/server/migrations/194_domain_event_seq_unique_index.up.sql new file mode 100644 index 00000000000..dc145cc70aa --- /dev/null +++ b/server/migrations/194_domain_event_seq_unique_index.up.sql @@ -0,0 +1,8 @@ +-- Single-statement CONCURRENTLY migration: Postgres rejects CREATE INDEX +-- CONCURRENTLY inside a transaction or a multi-command string, and the +-- migration runner execs each file as one command (see 080/135). +-- +-- Enforces the monotonic-uniqueness invariant of domain_event.seq and serves +-- the ordered `... ORDER BY seq` drain/scan the PR3 matcher will use. +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_seq + ON domain_event (seq); diff --git a/server/migrations/195_domain_event_dispatch_index.down.sql b/server/migrations/195_domain_event_dispatch_index.down.sql new file mode 100644 index 00000000000..80aa4ff5945 --- /dev/null +++ b/server/migrations/195_domain_event_dispatch_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_dispatch; diff --git a/server/migrations/195_domain_event_dispatch_index.up.sql b/server/migrations/195_domain_event_dispatch_index.up.sql new file mode 100644 index 00000000000..557c4b6b556 --- /dev/null +++ b/server/migrations/195_domain_event_dispatch_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 194). Backs the PR3 matcher's +-- claim scan for undispatched, now-available events in seq order: +-- WHERE dispatch_status = 'pending' AND available_at <= now() ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_dispatch + ON domain_event (dispatch_status, available_at, seq); diff --git a/server/migrations/196_domain_event_correlation_index.down.sql b/server/migrations/196_domain_event_correlation_index.down.sql new file mode 100644 index 00000000000..3bf1d98cdc5 --- /dev/null +++ b/server/migrations/196_domain_event_correlation_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_correlation; diff --git a/server/migrations/196_domain_event_correlation_index.up.sql b/server/migrations/196_domain_event_correlation_index.up.sql new file mode 100644 index 00000000000..fa1314a985b --- /dev/null +++ b/server/migrations/196_domain_event_correlation_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 194). Backs correlation-chain +-- reads (GET /api/events?correlation_id=) and loop/depth guardrail lookups: +-- WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_correlation + ON domain_event (workspace_id, correlation_id, seq); diff --git a/server/migrations/197_domain_event_type_index.down.sql b/server/migrations/197_domain_event_type_index.down.sql new file mode 100644 index 00000000000..08ae0ccb9b0 --- /dev/null +++ b/server/migrations/197_domain_event_type_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_type; diff --git a/server/migrations/197_domain_event_type_index.up.sql b/server/migrations/197_domain_event_type_index.up.sql new file mode 100644 index 00000000000..d54d8774880 --- /dev/null +++ b/server/migrations/197_domain_event_type_index.up.sql @@ -0,0 +1,5 @@ +-- Single-statement CONCURRENTLY migration (see 194). Backs per-workspace, +-- per-type event scans used by explain/debug tooling: +-- WHERE workspace_id = $1 AND type = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_type + ON domain_event (workspace_id, type, seq); diff --git a/server/migrations/198_domain_event_subject_index.down.sql b/server/migrations/198_domain_event_subject_index.down.sql new file mode 100644 index 00000000000..6a2a5d1ef06 --- /dev/null +++ b/server/migrations/198_domain_event_subject_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_domain_event_subject; diff --git a/server/migrations/198_domain_event_subject_index.up.sql b/server/migrations/198_domain_event_subject_index.up.sql new file mode 100644 index 00000000000..03d386916fe --- /dev/null +++ b/server/migrations/198_domain_event_subject_index.up.sql @@ -0,0 +1,6 @@ +-- Single-statement CONCURRENTLY migration (see 194). Backs "all events about +-- this subject" scans (a given issue / comment / task) for debug + the future +-- stage sensor: +-- WHERE subject_type = $1 AND subject_id = $2 ORDER BY seq +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_subject + ON domain_event (subject_type, subject_id, seq); diff --git a/server/pkg/db/generated/domain_event.sql.go b/server/pkg/db/generated/domain_event.sql.go new file mode 100644 index 00000000000..95b2662f190 --- /dev/null +++ b/server/pkg/db/generated/domain_event.sql.go @@ -0,0 +1,233 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: domain_event.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const countDomainEventsBySubject = `-- name: CountDomainEventsBySubject :one +SELECT count(*) FROM domain_event +WHERE subject_type = $1 + AND subject_id = $2 +` + +type CountDomainEventsBySubjectParams struct { + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` +} + +func (q *Queries) CountDomainEventsBySubject(ctx context.Context, arg CountDomainEventsBySubjectParams) (int64, error) { + row := q.db.QueryRow(ctx, countDomainEventsBySubject, arg.SubjectType, arg.SubjectID) + var count int64 + err := row.Scan(&count) + return count, err +} + +const createDomainEvent = `-- name: CreateDomainEvent :one + +INSERT INTO domain_event ( + id, + workspace_id, + type, + schema_version, + subject_type, + subject_id, + actor_type, + actor_id, + payload, + correlation_id, + causation_execution_id, + causation_action_index, + hop_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 +) +RETURNING id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at +` + +type CreateDomainEventParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID pgtype.UUID `json:"actor_id"` + Payload []byte `json:"payload"` + CorrelationID pgtype.UUID `json:"correlation_id"` + CausationExecutionID pgtype.UUID `json:"causation_execution_id"` + CausationActionIndex pgtype.Int4 `json:"causation_action_index"` + HopCount int32 `json:"hop_count"` +} + +// Transactional-outbox domain event log (MUL-4332 §4.1). CreateDomainEvent is +// the only write; callers invoke it on a *db.Queries already bound to their +// own pgx.Tx (WithTx) so the event commits atomically with the domain fact. +// dispatch_status ('pending'), available_at (now()), attempts (0), seq +// (nextval) and created_at (now()) all come from column defaults — this is a +// root outbox write, never a re-queue, so the writer never sets them. +func (q *Queries) CreateDomainEvent(ctx context.Context, arg CreateDomainEventParams) (DomainEvent, error) { + row := q.db.QueryRow(ctx, createDomainEvent, + arg.ID, + arg.WorkspaceID, + arg.Type, + arg.SchemaVersion, + arg.SubjectType, + arg.SubjectID, + arg.ActorType, + arg.ActorID, + arg.Payload, + arg.CorrelationID, + arg.CausationExecutionID, + arg.CausationActionIndex, + arg.HopCount, + ) + var i DomainEvent + err := row.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ) + return i, err +} + +const deleteDispatchedDomainEventsBefore = `-- name: DeleteDispatchedDomainEventsBefore :execrows +DELETE FROM domain_event +WHERE id IN ( + SELECT de.id FROM domain_event de + WHERE de.dispatch_status = 'dispatched' + AND de.created_at < $1 + ORDER BY de.seq + LIMIT $2 +) +` + +type DeleteDispatchedDomainEventsBeforeParams struct { + CreatedAt pgtype.Timestamptz `json:"created_at"` + Limit int32 `json:"limit"` +} + +// DeleteDispatchedDomainEventsBefore is the retention sweep (MUL-4332 §4.1): +// it only reclaims events already marked 'dispatched' and older than the TTL +// cutoff, in bounded batches so a large backlog never monopolizes the DB. In +// PR1 nothing dispatches events, so this is a no-op until the PR3 matcher runs. +func (q *Queries) DeleteDispatchedDomainEventsBefore(ctx context.Context, arg DeleteDispatchedDomainEventsBeforeParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteDispatchedDomainEventsBefore, arg.CreatedAt, arg.Limit) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getDomainEvent = `-- name: GetDomainEvent :one +SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event +WHERE id = $1 +` + +func (q *Queries) GetDomainEvent(ctx context.Context, id pgtype.UUID) (DomainEvent, error) { + row := q.db.QueryRow(ctx, getDomainEvent, id) + var i DomainEvent + err := row.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ) + return i, err +} + +const listDomainEventsByCorrelation = `-- name: ListDomainEventsByCorrelation :many +SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event +WHERE workspace_id = $1 + AND correlation_id = $2 +ORDER BY seq ASC +` + +type ListDomainEventsByCorrelationParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` +} + +func (q *Queries) ListDomainEventsByCorrelation(ctx context.Context, arg ListDomainEventsByCorrelationParams) ([]DomainEvent, error) { + rows, err := q.db.Query(ctx, listDomainEventsByCorrelation, arg.WorkspaceID, arg.CorrelationID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DomainEvent{} + for rows.Next() { + var i DomainEvent + if err := rows.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/server/pkg/db/generated/models.go b/server/pkg/db/generated/models.go index b7f277e5409..4efa7f9332f 100644 --- a/server/pkg/db/generated/models.go +++ b/server/pkg/db/generated/models.go @@ -452,6 +452,31 @@ type DaemonToken struct { CreatedAt pgtype.Timestamptz `json:"created_at"` } +// Transactional-outbox domain event log (MUL-4332 §4.1). One row per committed domain fact, written in the same tx as the fact. Source of truth for the hooks engine; no FK, no cascade, app-layer integrity. dispatch_* columns are the single-consumer outbox lease consumed by the PR3 matcher. +type DomainEvent struct { + ID pgtype.UUID `json:"id"` + Seq int64 `json:"seq"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` + SubjectType string `json:"subject_type"` + SubjectID pgtype.UUID `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID pgtype.UUID `json:"actor_id"` + Payload []byte `json:"payload"` + CorrelationID pgtype.UUID `json:"correlation_id"` + CausationExecutionID pgtype.UUID `json:"causation_execution_id"` + CausationActionIndex pgtype.Int4 `json:"causation_action_index"` + HopCount int32 `json:"hop_count"` + DispatchStatus string `json:"dispatch_status"` + Attempts int32 `json:"attempts"` + AvailableAt pgtype.Timestamptz `json:"available_at"` + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseExpiresAt pgtype.Timestamptz `json:"lease_expires_at"` + DispatchedAt pgtype.Timestamptz `json:"dispatched_at"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type Feedback struct { ID pgtype.UUID `json:"id"` UserID pgtype.UUID `json:"user_id"` diff --git a/server/pkg/db/queries/domain_event.sql b/server/pkg/db/queries/domain_event.sql new file mode 100644 index 00000000000..0fcfdc986b4 --- /dev/null +++ b/server/pkg/db/queries/domain_event.sql @@ -0,0 +1,55 @@ +-- Transactional-outbox domain event log (MUL-4332 §4.1). CreateDomainEvent is +-- the only write; callers invoke it on a *db.Queries already bound to their +-- own pgx.Tx (WithTx) so the event commits atomically with the domain fact. + +-- name: CreateDomainEvent :one +-- dispatch_status ('pending'), available_at (now()), attempts (0), seq +-- (nextval) and created_at (now()) all come from column defaults — this is a +-- root outbox write, never a re-queue, so the writer never sets them. +INSERT INTO domain_event ( + id, + workspace_id, + type, + schema_version, + subject_type, + subject_id, + actor_type, + actor_id, + payload, + correlation_id, + causation_execution_id, + causation_action_index, + hop_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 +) +RETURNING *; + +-- name: GetDomainEvent :one +SELECT * FROM domain_event +WHERE id = $1; + +-- name: ListDomainEventsByCorrelation :many +SELECT * FROM domain_event +WHERE workspace_id = $1 + AND correlation_id = $2 +ORDER BY seq ASC; + +-- name: CountDomainEventsBySubject :one +SELECT count(*) FROM domain_event +WHERE subject_type = $1 + AND subject_id = $2; + +-- DeleteDispatchedDomainEventsBefore is the retention sweep (MUL-4332 §4.1): +-- it only reclaims events already marked 'dispatched' and older than the TTL +-- cutoff, in bounded batches so a large backlog never monopolizes the DB. In +-- PR1 nothing dispatches events, so this is a no-op until the PR3 matcher runs. +-- name: DeleteDispatchedDomainEventsBefore :execrows +DELETE FROM domain_event +WHERE id IN ( + SELECT de.id FROM domain_event de + WHERE de.dispatch_status = 'dispatched' + AND de.created_at < $1 + ORDER BY de.seq + LIMIT $2 +); From 95f3fea42a6efed9fcabaad74b4aaa696863345f Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Wed, 15 Jul 2026 18:54:46 +0800 Subject: [PATCH 02/15] fix(automation): address Elon review on domain event outbox (MUL-4332 PR1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto main and resolved all six merge-blocking review points: 1. Renumber migrations 196–201 (193–195 are now issue_property on main); the TestMigrationNumericPrefixesStayUniqueAfterLegacySet lint passes again. 2. Converge the remaining v1 producers in-tx: agent-runtime and system child-done comments (comment.created), and all four bulk task.failed paths — the three runtime sweepers plus daemon orphan-recovery — through a shared FailTasksInTxWithEvents helper. No terminal path commits a fact without its event. 3. Read issue status/assignee under a row lock INSIDE the update tx so the event `from` is the true edge under concurrent transitions (single + batch UpdateIssue, GitHub merge close, task-failure reset), not a pre-tx snapshot. 4. Task-event workspace resolution is fail-closed: resolve via the full attribution chain (issue/chat/autopilot/quick-create/agent) on the tx handle, and roll the terminal transition back if unresolvable rather than skipping the event best-effort. 5. Retention is now an explicit no-op until PR3 — removed the weak dispatched+TTL delete that lacked the hook_execution terminal predicate. 6. Actor identity is fail-closed: ActorFrom no longer laundelrs an unknown type into system, and validate rejects a system actor carrying an id or a member/agent/hook actor missing one. Tests: concurrent status-edge correctness (-race), fail-closed workspace resolution, bulk task.failed emission via the shared helper, and expanded actor-validation cases. Full internal/service, internal/handler, internal/domainevent and migration-lint suites pass on a migrated DB. Co-authored-by: multica-agent --- server/cmd/server/domain_event_retention.go | 82 ++----- server/cmd/server/main.go | 6 +- server/cmd/server/runtime_sweeper.go | 29 ++- server/internal/domainevent/domainevent.go | 13 +- server/internal/domainevent/event.go | 11 + server/internal/domainevent/event_test.go | 41 +++- .../handler/domain_event_outbox_test.go | 61 ++++++ server/internal/handler/github.go | 13 +- server/internal/handler/issue.go | 50 +++-- server/internal/handler/issue_child_done.go | 34 ++- server/internal/handler/task_lifecycle.go | 7 +- .../service/domain_event_task_failed_test.go | 74 +++++++ .../domain_event_task_workspace_test.go | 46 ++++ server/internal/service/task.go | 200 +++++++++++++----- ...ent.down.sql => 196_domain_event.down.sql} | 0 ...n_event.up.sql => 196_domain_event.up.sql} | 2 +- ...97_domain_event_seq_unique_index.down.sql} | 0 ... 197_domain_event_seq_unique_index.up.sql} | 0 ... 198_domain_event_dispatch_index.down.sql} | 0 ...=> 198_domain_event_dispatch_index.up.sql} | 2 +- ...9_domain_event_correlation_index.down.sql} | 0 ...199_domain_event_correlation_index.up.sql} | 2 +- ...l => 200_domain_event_type_index.down.sql} | 0 ...sql => 200_domain_event_type_index.up.sql} | 2 +- ...> 201_domain_event_subject_index.down.sql} | 0 ... => 201_domain_event_subject_index.up.sql} | 2 +- server/pkg/db/generated/domain_event.sql.go | 28 --- server/pkg/db/generated/issue.sql.go | 31 +++ server/pkg/db/queries/domain_event.sql | 19 +- server/pkg/db/queries/issue.sql | 12 ++ 30 files changed, 549 insertions(+), 218 deletions(-) create mode 100644 server/internal/service/domain_event_task_failed_test.go create mode 100644 server/internal/service/domain_event_task_workspace_test.go rename server/migrations/{193_domain_event.down.sql => 196_domain_event.down.sql} (100%) rename server/migrations/{193_domain_event.up.sql => 196_domain_event.up.sql} (98%) rename server/migrations/{194_domain_event_seq_unique_index.down.sql => 197_domain_event_seq_unique_index.down.sql} (100%) rename server/migrations/{194_domain_event_seq_unique_index.up.sql => 197_domain_event_seq_unique_index.up.sql} (100%) rename server/migrations/{195_domain_event_dispatch_index.down.sql => 198_domain_event_dispatch_index.down.sql} (100%) rename server/migrations/{195_domain_event_dispatch_index.up.sql => 198_domain_event_dispatch_index.up.sql} (81%) rename server/migrations/{196_domain_event_correlation_index.down.sql => 199_domain_event_correlation_index.down.sql} (100%) rename server/migrations/{196_domain_event_correlation_index.up.sql => 199_domain_event_correlation_index.up.sql} (81%) rename server/migrations/{197_domain_event_type_index.down.sql => 200_domain_event_type_index.down.sql} (100%) rename server/migrations/{197_domain_event_type_index.up.sql => 200_domain_event_type_index.up.sql} (78%) rename server/migrations/{198_domain_event_subject_index.down.sql => 201_domain_event_subject_index.down.sql} (100%) rename server/migrations/{198_domain_event_subject_index.up.sql => 201_domain_event_subject_index.up.sql} (81%) diff --git a/server/cmd/server/domain_event_retention.go b/server/cmd/server/domain_event_retention.go index c1f39aef101..40f6e73baa1 100644 --- a/server/cmd/server/domain_event_retention.go +++ b/server/cmd/server/domain_event_retention.go @@ -3,72 +3,22 @@ package main import ( "context" "log/slog" - "time" - - "github.com/jackc/pgx/v5/pgtype" - db "github.com/multica-ai/multica/server/pkg/db/generated" -) - -// Domain event retention (MUL-4332 §4.1 / §9). The transactional-outbox -// domain_event table would grow without bound, so a periodic sweep reclaims -// events that are BOTH already dispatched AND older than the TTL. In PR1 there -// is no matcher, so nothing is ever marked 'dispatched' — this sweep is a -// deliberate no-op until PR3, matching the "zero behavior change" contract -// while ensuring retention lands with the table it governs. -const ( - // domainEventRetentionInterval is how often the sweep runs. Retention is - // coarse-grained, so an hourly tick is plenty. - domainEventRetentionInterval = 1 * time.Hour - // domainEventTTL is the retention window: dispatched events older than this - // are reclaimed (MUL-4332 §9 fixes this at 90 days). - domainEventTTL = 90 * 24 * time.Hour - // domainEventRetentionBatch bounds a single DELETE so a large backlog can - // never monopolize the DB; the sweep drains in batches until a short one. - domainEventRetentionBatch = 1000 ) -// runDomainEventRetention runs the retention sweep on a ticker until ctx is -// cancelled. Registered alongside the other sweepCtx-bound workers in main so -// it stops cleanly on shutdown. -func runDomainEventRetention(ctx context.Context, queries *db.Queries) { - ticker := time.NewTicker(domainEventRetentionInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepDomainEvents(ctx, queries) - } - } -} - -// sweepDomainEvents deletes dispatched-and-expired events in bounded batches. -func sweepDomainEvents(ctx context.Context, queries *db.Queries) { - cutoff := pgtype.Timestamptz{Time: time.Now().Add(-domainEventTTL), Valid: true} - var total int64 - for { - deleted, err := queries.DeleteDispatchedDomainEventsBefore(ctx, db.DeleteDispatchedDomainEventsBeforeParams{ - CreatedAt: cutoff, - Limit: domainEventRetentionBatch, - }) - if err != nil { - slog.Warn("domain event retention: delete failed", "error", err) - return - } - total += deleted - // A short batch means the backlog is drained. - if deleted < domainEventRetentionBatch { - break - } - // Bail out promptly on shutdown between batches. - select { - case <-ctx.Done(): - return - default: - } - } - if total > 0 { - slog.Info("domain event retention: reclaimed dispatched events", "count", total) - } +// Domain event retention (MUL-4332 §4.1 / §9) is DELIBERATELY an explicit no-op +// in PR1. The correct retention predicate is "dispatched AND older than the TTL +// AND every related hook_execution is terminal" — but hook_execution does not +// exist until PR3, and PR1 has no dispatcher, so nothing is ever eligible for +// deletion. Shipping a weaker "dispatched + TTL" sweep now would, the moment PR3 +// flips dispatching on, risk reclaiming events whose executions are still +// running or retrying (review point 5). Retention therefore lands in PR3 +// alongside hook_execution, the full terminal predicate, and concurrent-sweeper +// tests. +// +// The worker is kept wired (rather than silently omitted) so the intent is +// visible at the call site: it logs once and then idles until shutdown, doing no +// deletes. +func runDomainEventRetention(ctx context.Context) { + slog.Info("domain event retention: explicit no-op in PR1, deferred to PR3 (needs hook_execution terminal predicate)") + <-ctx.Done() } diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index c17787df92f..dd4678bfe99 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -386,9 +386,9 @@ func main() { // Start background sweeper to mark stale runtimes as offline. go runRuntimeSweeper(sweepCtx, queries, liveness, taskSvc, bus) - // Reclaim dispatched-and-expired domain events (MUL-4332). No-op until the - // PR3 matcher marks events dispatched. - go runDomainEventRetention(sweepCtx, queries) + // Domain event retention (MUL-4332) — explicit no-op in PR1, wired for + // visibility; the real sweep lands in PR3 with hook_execution. + go runDomainEventRetention(sweepCtx) go heartbeatScheduler.Run(sweepCtx) go runAutopilotFailureMonitor(autopilotCtx, queries, bus, envFailureMonitorConfig()) go runDBStatsLogger(sweepCtx, pool) diff --git a/server/cmd/server/runtime_sweeper.go b/server/cmd/server/runtime_sweeper.go index 941e01c13e3..0261d1c3c57 100644 --- a/server/cmd/server/runtime_sweeper.go +++ b/server/cmd/server/runtime_sweeper.go @@ -168,7 +168,10 @@ func sweepStaleRuntimes(ctx context.Context, queries *db.Queries, liveness handl slog.Info("runtime sweeper: marked stale runtimes offline", "count", len(staleRows), "workspaces", len(workspaces)) // Fail orphaned tasks (dispatched/running) whose runtimes just went offline. - failedTasks, err := queries.FailTasksForOfflineRuntimes(ctx) + // Emit task.failed atomically with the bulk fail (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.FailTasksForOfflineRuntimes(ctx) + }) if err != nil { slog.Warn("runtime sweeper: failed to clean up stale tasks", "error", err) } else if len(failedTasks) > 0 { @@ -269,12 +272,15 @@ func gcRuntimes(ctx context.Context, queries *db.Queries, bus *events.Bus) { // edge where a runtime row lingers online-with-stale-heartbeat past the // wall clock (MUL-4107). func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService, bus *events.Bus) { - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ - DispatchTimeoutSecs: dispatchTimeoutSeconds, - RunningTimeoutSecs: runningTimeoutSeconds, - // Reuse the runtime stale window so the running-task backstop - // exactly matches what sweepStaleRuntimes considers "not alive". - RuntimeStaleSecs: staleThresholdSeconds, + // Emit task.failed atomically with the bulk fail (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.FailStaleTasks(ctx, db.FailStaleTasksParams{ + DispatchTimeoutSecs: dispatchTimeoutSeconds, + RunningTimeoutSecs: runningTimeoutSeconds, + // Reuse the runtime stale window so the running-task backstop + // exactly matches what sweepStaleRuntimes considers "not alive". + RuntimeStaleSecs: staleThresholdSeconds, + }) }) if err != nil { slog.Warn("task sweeper: failed to clean up stale tasks", "error", err) @@ -296,9 +302,12 @@ func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service. // a task is already queued. Capped to queuedExpireBatchSize per tick so a // big backlog can't monopolise the DB. func sweepExpiredQueuedTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService) { - failedTasks, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ - TtlSecs: queuedTTLSeconds, - MaxPerTick: queuedExpireBatchSize, + // Emit task.failed atomically with the bulk expiry (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ + TtlSecs: queuedTTLSeconds, + MaxPerTick: queuedExpireBatchSize, + }) }) if err != nil { slog.Warn("task sweeper: failed to expire stale queued tasks", "error", err) diff --git a/server/internal/domainevent/domainevent.go b/server/internal/domainevent/domainevent.go index 03ff25d3d7c..2b8d43e995e 100644 --- a/server/internal/domainevent/domainevent.go +++ b/server/internal/domainevent/domainevent.go @@ -105,14 +105,15 @@ func HookActor(id pgtype.UUID) Actor { return Actor{Type: ActorHook, ID: id} } func SystemActor() Actor { return Actor{Type: ActorSystem} } // ActorFrom builds an Actor from a raw (type, id) pair — for call sites that -// carry an existing creator_type/creator_id or author_type/author_id. An empty -// or unknown type degrades to a system actor so a mislabelled caller can never -// fabricate a member/agent identity. +// carry an existing creator_type/creator_id or author_type/author_id. +// +// It is fail-closed (MUL-4332 review point 6): a system actor is normalised to +// carry no id, but an unknown type is NOT silently degraded to system — it is +// passed through unchanged so Event.validate rejects it and the transaction +// aborts, rather than permanently recording a mislabelled actor as system. func ActorFrom(actorType string, id pgtype.UUID) Actor { - if !validActorTypes[actorType] { - return SystemActor() - } if actorType == ActorSystem { + // A system actor never carries an id; drop a stray one. return SystemActor() } return Actor{Type: actorType, ID: id} diff --git a/server/internal/domainevent/event.go b/server/internal/domainevent/event.go index 0dfacafdd07..c1054c38699 100644 --- a/server/internal/domainevent/event.go +++ b/server/internal/domainevent/event.go @@ -78,6 +78,17 @@ func (e Event) validate() error { if !validActorTypes[e.ActorType] { return fmt.Errorf("domainevent: invalid actor_type %q", e.ActorType) } + // Fail-closed actor identity (MUL-4332 review point 6): a system actor must + // carry NO id, and every other actor type must carry a valid one — so a + // dropped / unparsable id can never be silently recorded as a null or + // system actor. The caller's transaction aborts instead. + if e.ActorType == ActorSystem { + if e.ActorID.Valid { + return fmt.Errorf("domainevent: %s: system actor must not carry an actor_id", e.Type) + } + } else if !e.ActorID.Valid { + return fmt.Errorf("domainevent: %s: %s actor requires a valid actor_id", e.Type, e.ActorType) + } if !e.WorkspaceID.Valid { return fmt.Errorf("domainevent: %s missing workspace_id", e.Type) } diff --git a/server/internal/domainevent/event_test.go b/server/internal/domainevent/event_test.go index b4cb094e37c..10fbb73ff8c 100644 --- a/server/internal/domainevent/event_test.go +++ b/server/internal/domainevent/event_test.go @@ -77,6 +77,10 @@ func TestValidateRejectsBadEnvelopes(t *testing.T) { "wrong schema ver": mutate(func(e *Event) { e.SchemaVersion = 2 }), "wrong subject type": mutate(func(e *Event) { e.SubjectType = SubjectTask }), "bad actor type": mutate(func(e *Event) { e.ActorType = "wizard" }), + "member without id": mutate(func(e *Event) { e.ActorType = ActorMember; e.ActorID = pgtype.UUID{} }), + "agent without id": mutate(func(e *Event) { e.ActorType = ActorAgent; e.ActorID = pgtype.UUID{} }), + "system with id": mutate(func(e *Event) { e.ActorType = ActorSystem; e.ActorID = testUUID(t) }), + "unknown type w/ id": mutate(func(e *Event) { e.ActorType = "wizard"; e.ActorID = testUUID(t) }), "missing workspace": mutate(func(e *Event) { e.WorkspaceID = pgtype.UUID{} }), "missing subject": mutate(func(e *Event) { e.SubjectID = pgtype.UUID{} }), "empty payload": mutate(func(e *Event) { e.Payload = nil }), @@ -90,23 +94,40 @@ func TestValidateRejectsBadEnvelopes(t *testing.T) { } } -// ActorFrom must never let an unknown or empty actor type mint a member/agent -// identity — it degrades to system so a mislabelled caller can't fabricate -// authorship (guards the accountable-actor split from MUL-4332 §8). -func TestActorFromDegradesUnknownToSystem(t *testing.T) { +// ActorFrom is fail-closed (MUL-4332 review point 6): a system actor is +// normalised to carry no id, but an unknown/empty type is passed through +// unchanged so validate rejects it — it must NOT be silently laundered into a +// system actor, which would permanently mis-record authorship. +func TestActorFromFailClosed(t *testing.T) { id := testUUID(t) + ws, subj := testUUID(t), testUUID(t) + build := func(a Actor) error { + return IssueStatusChanged(ws, subj, a, IssueStatusChangedPayload{From: "a", To: "b"}).validate() + } + if a := ActorFrom("member", id); a.Type != ActorMember || a.ID != id { t.Errorf("member: got %+v", a) } - if a := ActorFrom("", id); a.Type != ActorSystem || a.ID.Valid { - t.Errorf("empty type should degrade to system with no id, got %+v", a) - } - if a := ActorFrom("wizard", id); a.Type != ActorSystem || a.ID.Valid { - t.Errorf("unknown type should degrade to system with no id, got %+v", a) - } if a := ActorFrom("system", id); a.Type != ActorSystem || a.ID.Valid { t.Errorf("system actor never carries an id, got %+v", a) } + // Unknown / empty types survive into the envelope and are rejected there. + if a := ActorFrom("wizard", id); a.Type != "wizard" { + t.Errorf("unknown type must pass through, got %+v", a) + } + if err := build(ActorFrom("wizard", id)); err == nil { + t.Error("unknown actor type must fail validate, not degrade to system") + } + if err := build(ActorFrom("", id)); err == nil { + t.Error("empty actor type must fail validate") + } + // The valid shapes still pass. + if err := build(MemberActor(id)); err != nil { + t.Errorf("member actor should validate: %v", err) + } + if err := build(SystemActor()); err != nil { + t.Errorf("system actor should validate: %v", err) + } } // The payload JSON shape is a wire contract the PR3 matcher's fixed-vocabulary diff --git a/server/internal/handler/domain_event_outbox_test.go b/server/internal/handler/domain_event_outbox_test.go index 1a2ad91343c..5cf9888ce43 100644 --- a/server/internal/handler/domain_event_outbox_test.go +++ b/server/internal/handler/domain_event_outbox_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "sync" "testing" "github.com/multica-ai/multica/server/internal/domainevent" @@ -129,3 +130,63 @@ func TestOutboxEmittedByIssueHandlers(t *testing.T) { t.Errorf("expected an issue.assigned event, got %+v", all) } } + +// Two concurrent status transitions on the same issue must each record the TRUE +// edge, not both read the same pre-transition snapshot (MUL-4332 review point +// 3). Because the event `from` is now read under a row lock inside the update +// tx, the two updates serialize and their events chain (todo→A, A→B) instead of +// both reporting from="todo". +func TestOutboxStatusFromCorrectUnderConcurrency(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + issueID := createTestIssue(t, "outbox concurrency "+t.Name(), "todo", "none") + t.Cleanup(func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // Fire two different transitions concurrently. FOR UPDATE serializes them. + statuses := []string{"in_progress", "done"} + var wg sync.WaitGroup + for _, st := range statuses { + wg.Add(1) + go func(status string) { + defer wg.Done() + w := httptest.NewRecorder() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{"status": status}) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(w, req) + }(st) + } + wg.Wait() + + var changes []outboxRow + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, issueID) { + if e.Type == domainevent.TypeIssueStatusChanged { + changes = append(changes, e) + } + } + if len(changes) != 2 { + t.Fatalf("expected 2 status_changed events, got %d: %+v", len(changes), changes) + } + + // The froms must be distinct (the bug produced two from="todo"); one edge + // starts at "todo" and the other starts where the first landed (a valid + // chain), regardless of which transition won the lock first. + from0 := payloadField(t, changes[0].Payload, "from") + from1 := payloadField(t, changes[1].Payload, "from") + if from0 == from1 { + t.Fatalf("both events share from=%q — stale snapshot bug (review point 3): %+v", from0, changes) + } + // Identify the todo-rooted edge and assert the other edge chains off its `to`. + byFrom := map[string]outboxRow{from0: changes[0], from1: changes[1]} + todoEdge, ok := byFrom["todo"] + if !ok { + t.Fatalf("expected one edge to start at todo, got froms %q/%q", from0, from1) + } + firstTo := payloadField(t, todoEdge.Payload, "to") + if _, chained := byFrom[firstTo]; !chained { + t.Errorf("edges do not chain: todo→%s but no event starts from %s (%+v)", firstTo, firstTo, changes) + } +} diff --git a/server/internal/handler/github.go b/server/internal/handler/github.go index 9fd20f05122..c4d085e58cd 100644 --- a/server/internal/handler/github.go +++ b/server/internal/handler/github.go @@ -1368,6 +1368,15 @@ func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, worksp // already-done issue produces no event). var updated db.Issue if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + // Lock + read the authoritative status so the event `from` is correct + // even if another writer moved the issue concurrently (review point 3). + before, err := qtx.LockIssueStatusForEvent(ctx, db.LockIssueStatusForEventParams{ + ID: issue.ID, + WorkspaceID: issue.WorkspaceID, + }) + if err != nil { + return nil, err + } u, err := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ ID: issue.ID, Status: "done", @@ -1377,11 +1386,11 @@ func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, worksp return nil, err } updated = u - if issue.Status == u.Status { + if before.Status == u.Status { return nil, nil } return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), - domainevent.IssueStatusChangedPayload{From: issue.Status, To: u.Status})}, nil + domainevent.IssueStatusChangedPayload{From: before.Status, To: u.Status})}, nil }); err != nil { slog.Warn("github: advance issue to done failed", "err", err) return diff --git a/server/internal/handler/issue.go b/server/internal/handler/issue.go index 676e1d263c7..a0e12d96982 100644 --- a/server/internal/handler/issue.go +++ b/server/internal/handler/issue.go @@ -2696,10 +2696,18 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { // Transactional outbox (MUL-4332): commit the issue update and any derived // status_changed / assigned event in one transaction, so a crash can never - // separate the fact from its event. The events are computed by diffing - // prevIssue against the freshly-updated row. + // separate the fact from its event. The event `from` is read under a row + // lock INSIDE the tx (not from the pre-tx prevIssue snapshot), so concurrent + // transitions serialize and each records the true edge (review point 3). var issue db.Issue err = domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + before, err := qtx.LockIssueStatusForEvent(r.Context(), db.LockIssueStatusForEventParams{ + ID: prevIssue.ID, + WorkspaceID: prevIssue.WorkspaceID, + }) + if err != nil { + return nil, err + } updated, err := qtx.UpdateIssue(r.Context(), params) if err != nil { return nil, err @@ -2707,16 +2715,19 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { issue = updated var events []domainevent.Event - if req.Status != nil && prevIssue.Status != updated.Status { + if before.Status != updated.Status { events = append(events, domainevent.IssueStatusChanged(updated.WorkspaceID, updated.ID, eventActor, - domainevent.IssueStatusChangedPayload{From: prevIssue.Status, To: updated.Status})) + domainevent.IssueStatusChangedPayload{From: before.Status, To: updated.Status})) } - if (req.AssigneeType != nil || req.AssigneeID != nil) && - (prevIssue.AssigneeType.String != updated.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(updated.AssigneeID)) { + // Only emit assigned when the request actually targeted the assignee, so + // an unrelated field update never surfaces a spurious assignment; `from` + // still comes from the locked row. + if (touchedType || touchedID) && + (before.AssigneeType.String != updated.AssigneeType.String || uuidToString(before.AssigneeID) != uuidToString(updated.AssigneeID)) { events = append(events, domainevent.IssueAssigned(updated.WorkspaceID, updated.ID, eventActor, domainevent.IssueAssignedPayload{ - FromAssigneeType: prevIssue.AssigneeType.String, - FromAssigneeID: uuidToString(prevIssue.AssigneeID), + FromAssigneeType: before.AssigneeType.String, + FromAssigneeID: uuidToString(before.AssigneeID), ToAssigneeType: updated.AssigneeType.String, ToAssigneeID: uuidToString(updated.AssigneeID), })) @@ -3268,25 +3279,34 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { batchEventActor := domainevent.ActorFrom(actorType, batchActorUUID) // Transactional outbox (MUL-4332): each row's update and its derived - // status_changed / assigned events commit atomically, per issue. + // status_changed / assigned events commit atomically, per issue. `from` + // is read under a row lock inside the tx so concurrent transitions record + // the true edge (review point 3). var issue db.Issue if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + before, err := qtx.LockIssueStatusForEvent(r.Context(), db.LockIssueStatusForEventParams{ + ID: prevIssue.ID, + WorkspaceID: prevIssue.WorkspaceID, + }) + if err != nil { + return nil, err + } updatedIssue, err := qtx.UpdateIssue(r.Context(), params) if err != nil { return nil, err } issue = updatedIssue var events []domainevent.Event - if req.Updates.Status != nil && prevIssue.Status != updatedIssue.Status { + if before.Status != updatedIssue.Status { events = append(events, domainevent.IssueStatusChanged(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, - domainevent.IssueStatusChangedPayload{From: prevIssue.Status, To: updatedIssue.Status})) + domainevent.IssueStatusChangedPayload{From: before.Status, To: updatedIssue.Status})) } - if (req.Updates.AssigneeType != nil || req.Updates.AssigneeID != nil) && - (prevIssue.AssigneeType.String != updatedIssue.AssigneeType.String || uuidToString(prevIssue.AssigneeID) != uuidToString(updatedIssue.AssigneeID)) { + if (batchTouchedType || batchTouchedID) && + (before.AssigneeType.String != updatedIssue.AssigneeType.String || uuidToString(before.AssigneeID) != uuidToString(updatedIssue.AssigneeID)) { events = append(events, domainevent.IssueAssigned(updatedIssue.WorkspaceID, updatedIssue.ID, batchEventActor, domainevent.IssueAssignedPayload{ - FromAssigneeType: prevIssue.AssigneeType.String, - FromAssigneeID: uuidToString(prevIssue.AssigneeID), + FromAssigneeType: before.AssigneeType.String, + FromAssigneeID: uuidToString(before.AssigneeID), ToAssigneeType: updatedIssue.AssigneeType.String, ToAssigneeID: uuidToString(updatedIssue.AssigneeID), })) diff --git a/server/internal/handler/issue_child_done.go b/server/internal/handler/issue_child_done.go index b9f0be80c9a..5861efa5ecd 100644 --- a/server/internal/handler/issue_child_done.go +++ b/server/internal/handler/issue_child_done.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/domainevent" db "github.com/multica-ai/multica/server/pkg/db/generated" "github.com/multica-ai/multica/server/pkg/protocol" ) @@ -300,16 +301,29 @@ func (h *Handler) postChildDoneComment(ctx context.Context, parent, completed db // author_type='system', author_id=zero UUID. The zero UUID is a valid 16 // byte value and the column is NOT NULL; frontend code should branch on // author_type === 'system' rather than on the UUID value. - comment, err := h.Queries.CreateComment(ctx, db.CreateCommentParams{ - IssueID: parent.ID, - WorkspaceID: parent.WorkspaceID, - AuthorType: "system", - AuthorID: pgtype.UUID{Valid: true}, - Content: content, - Type: "system", - ParentID: pgtype.UUID{Valid: false}, - }) - if err != nil { + // Transactional outbox (MUL-4332 review point 2): the system child-done + // comment and its comment.created event commit in one transaction. + var comment db.Comment + if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, cErr := qtx.CreateComment(ctx, db.CreateCommentParams{ + IssueID: parent.ID, + WorkspaceID: parent.WorkspaceID, + AuthorType: "system", + AuthorID: pgtype.UUID{Valid: true}, + Content: content, + Type: "system", + ParentID: pgtype.UUID{Valid: false}, + }) + if cErr != nil { + return nil, cErr + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, domainevent.SystemActor(), + domainevent.CommentCreatedPayload{ + IssueID: uuidToString(created.IssueID), + AuthorType: "system", + })}, nil + }); err != nil { slog.Warn("child done: create system comment failed", "error", err, "child_id", childID, diff --git a/server/internal/handler/task_lifecycle.go b/server/internal/handler/task_lifecycle.go index e84abcb4200..3013ea50d89 100644 --- a/server/internal/handler/task_lifecycle.go +++ b/server/internal/handler/task_lifecycle.go @@ -29,7 +29,12 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { return } - rows, err := h.Queries.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID)) + // Emit task.failed atomically with the bulk orphan-fail (MUL-4332 review + // point 2), so daemon-driven recovery converges onto the outbox like the + // runtime sweepers. + rows, err := h.TaskService.FailTasksInTxWithEvents(r.Context(), func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID)) + }) if err != nil { slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) writeError(w, http.StatusInternalServerError, "recover orphans failed") diff --git a/server/internal/service/domain_event_task_failed_test.go b/server/internal/service/domain_event_task_failed_test.go new file mode 100644 index 00000000000..bdbc23a5778 --- /dev/null +++ b/server/internal/service/domain_event_task_failed_test.go @@ -0,0 +1,74 @@ +package service + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// FailTasksInTxWithEvents is the shared mechanism behind every bulk task.failed +// path (the three runtime sweepers + daemon orphan recovery). Failing a task +// through it must persist a task.failed domain event atomically with the fail +// (MUL-4332 review point 2), stamped with the resolved workspace and issue. +func TestFailTasksInTxWithEventsEmitsTaskFailed(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + // Seed a running task for that agent/issue. + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed running task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + failed, err := svc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, ferr := qtx.FailAgentTask(ctx, db.FailAgentTaskParams{ + ID: util.MustParseUUID(taskID), + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + if ferr != nil { + return nil, ferr + } + return []db.AgentTaskQueue{row}, nil + }) + if err != nil { + t.Fatalf("FailTasksInTxWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("expected 1 failed task, got %d", len(failed)) + } + + // Exactly one task.failed event for the task, carrying the issue + error. + var evtType, payload string + if err := pool.QueryRow(ctx, + `SELECT type, payload::text FROM domain_event WHERE subject_type = 'task' AND subject_id = $1`, + taskID).Scan(&evtType, &payload); err != nil { + t.Fatalf("expected a task.failed domain event: %v", err) + } + if evtType != "task.failed" { + t.Errorf("type = %q, want task.failed", evtType) + } + if !strings.Contains(payload, issueID) { + t.Errorf("payload %s should carry issue_id %s", payload, issueID) + } + if !strings.Contains(payload, "runtime_offline") { + t.Errorf("payload %s should carry the failure reason", payload) + } +} diff --git a/server/internal/service/domain_event_task_workspace_test.go b/server/internal/service/domain_event_task_workspace_test.go new file mode 100644 index 00000000000..425c92d5619 --- /dev/null +++ b/server/internal/service/domain_event_task_workspace_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// resolveTaskWorkspaceForEvent must be fail-closed (MUL-4332 review point 4): a +// task with no resolvable attribution returns an invalid UUID, which the +// task-terminal callers treat as an error and roll the transition back rather +// than committing a fact with no event. It must also succeed via the agent +// fallback for a task that carries only an agent. +func TestResolveTaskWorkspaceForEventFailClosed(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + // Unresolvable: no issue / chat / autopilot / quick-create link and a bogus + // agent id that matches no row → invalid workspace. + orphan := db.AgentTaskQueue{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + AgentID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + } + if ws := svc.resolveTaskWorkspaceForEvent(ctx, queries, orphan); ws.Valid { + t.Fatalf("expected unresolvable workspace to be invalid, got %s", util.UUIDToString(ws)) + } + + // Resolvable via the agent fallback: a task carrying a real agent resolves to + // that agent's workspace. + agentID := createClaimCapacityFixture(t, ctx, pool) + task := db.AgentTaskQueue{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, + AgentID: util.MustParseUUID(agentID), + } + if ws := svc.resolveTaskWorkspaceForEvent(ctx, queries, task); !ws.Valid { + t.Fatalf("expected agent fallback to resolve a workspace for agent %s", agentID) + } +} diff --git a/server/internal/service/task.go b/server/internal/service/task.go index 465d633f708..ca34469fff4 100644 --- a/server/internal/service/task.go +++ b/server/internal/service/task.go @@ -2630,15 +2630,20 @@ func (s *TaskService) CompleteTask(ctx context.Context, taskID pgtype.UUID, resu // Transactional outbox (MUL-4332): emit task.completed atomically with // the status flip. The status CAS above makes this exactly-once — a // replayed callback finds the task already terminal and never re-emits. - if wsID := s.taskWorkspaceID(ctx, qtx, t.AgentID); wsID.Valid { - evt := domainevent.TaskCompleted(wsID, t.ID, domainevent.AgentActor(t.AgentID), - domainevent.TaskCompletedPayload{ - IssueID: util.UUIDToString(t.IssueID), - AgentID: util.UUIDToString(t.AgentID), - }) - if _, err := domainevent.Write(ctx, qtx, evt); err != nil { - return fmt.Errorf("write task.completed event: %w", err) - } + // Fail-closed on workspace resolution (review point 4): if the workspace + // is unresolvable we error out and roll the completion back rather than + // commit the fact without its event. + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + return fmt.Errorf("task.completed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + } + evt := domainevent.TaskCompleted(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskCompletedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.completed event: %w", err) } return nil }); err != nil { @@ -2971,18 +2976,21 @@ func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, // Transactional outbox (MUL-4332): emit task.failed atomically with the // fail (and any retry child). retryable reflects whether this failure - // spawned an auto-retry; the status CAS makes it exactly-once. - if wsID := s.taskWorkspaceID(ctx, qtx, t.AgentID); wsID.Valid { - evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), - domainevent.TaskFailedPayload{ - IssueID: util.UUIDToString(t.IssueID), - AgentID: util.UUIDToString(t.AgentID), - Retryable: wantRetry, - ErrorCode: failureReason, - }) - if _, err := domainevent.Write(ctx, qtx, evt); err != nil { - return fmt.Errorf("write task.failed event: %w", err) - } + // spawned an auto-retry; the status CAS makes it exactly-once. Fail-closed + // on workspace resolution (review point 4). + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + return fmt.Errorf("task.failed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + } + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskFailedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + Retryable: wantRetry, + ErrorCode: failureReason, + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return fmt.Errorf("write task.failed event: %w", err) } return nil }); err != nil { @@ -3466,6 +3474,15 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas // Transactional outbox (MUL-4332): the stuck-issue reset bypasses the // HTTP UpdateIssue path, so emit issue.status_changed atomically here. updateErr := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + // Lock + read the true current status inside the tx so the event + // `from` is correct under concurrent transitions (review point 3). + before, bErr := qtx.LockIssueStatusForEvent(ctx, db.LockIssueStatusForEventParams{ + ID: t.IssueID, + WorkspaceID: issue.WorkspaceID, + }) + if bErr != nil { + return nil, bErr + } u, uErr := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ ID: t.IssueID, Status: "todo", @@ -3475,8 +3492,11 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas return nil, uErr } updatedIssue = u + if before.Status == u.Status { + return nil, nil + } return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), - domainevent.IssueStatusChangedPayload{From: issue.Status, To: u.Status})}, nil + domainevent.IssueStatusChangedPayload{From: before.Status, To: u.Status})}, nil }) if updateErr != nil { slog.Warn("handle failed tasks: reset stuck issue failed", @@ -3527,24 +3547,91 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas // runInTx executes fn inside a single DB transaction. If TxStarter is nil // (e.g. some tests construct TaskService directly), fn runs against the // regular Queries handle without transactional guarantees. -// taskWorkspaceID resolves the workspace of a task via its agent, to stamp the -// workspace_id of a task domain event (agent_task_queue itself carries no -// workspace column). It reads on the passed qtx so the lookup stays in the -// caller's transaction. Best-effort: an unresolvable agent (already deleted) -// returns an invalid UUID and the caller skips the event rather than wedging -// the task's terminal callback — a missing agent means the workspace is -// genuinely unknowable, and the lost event is bounded to that pathological case. -func (s *TaskService) taskWorkspaceID(ctx context.Context, qtx *db.Queries, agentID pgtype.UUID) pgtype.UUID { - if !agentID.Valid { - return pgtype.UUID{} - } - agent, err := qtx.GetAgent(ctx, agentID) +// resolveTaskWorkspaceForEvent resolves a task's workspace for stamping a task +// domain event (agent_task_queue carries no workspace column). It reads on the +// passed qtx so the lookup stays in the caller's transaction, and walks the +// task's stable attribution — issue, chat session, autopilot run, quick-create +// context — before finally falling back to the owning agent. This mirrors +// ResolveTaskWorkspaceID's chain but tx-scoped and with the agent fallback that +// resolver lacks. An invalid return means genuinely unresolvable; the caller +// treats that as fail-closed (review point 4): it errors out and rolls the whole +// terminal transition back rather than committing a fact with no event. +func (s *TaskService) resolveTaskWorkspaceForEvent(ctx context.Context, qtx *db.Queries, task db.AgentTaskQueue) pgtype.UUID { + if task.IssueID.Valid { + if issue, err := qtx.GetIssue(ctx, task.IssueID); err == nil { + return issue.WorkspaceID + } + } + if task.ChatSessionID.Valid { + if cs, err := qtx.GetChatSession(ctx, task.ChatSessionID); err == nil { + return cs.WorkspaceID + } + } + if task.AutopilotRunID.Valid { + if run, err := qtx.GetAutopilotRun(ctx, task.AutopilotRunID); err == nil { + if ap, err := qtx.GetAutopilot(ctx, run.AutopilotID); err == nil { + return ap.WorkspaceID + } + } + } + if qc, ok := s.parseQuickCreateContext(task); ok { + if ws, err := util.ParseUUID(qc.WorkspaceID); err == nil && ws.Valid { + return ws + } + } + if task.AgentID.Valid { + if agent, err := qtx.GetAgent(ctx, task.AgentID); err == nil { + return agent.WorkspaceID + } + } + return pgtype.UUID{} +} + +// FailTasksInTxWithEvents runs a bulk-fail query inside a transaction and emits +// a task.failed domain event for each returned task atomically with the bulk +// UPDATE (MUL-4332 review point 2). The runtime sweepers and daemon +// orphan-recovery all funnel through this, so no bulk terminal path commits a +// fail without its event. It is all-or-nothing per batch: an event-write or +// workspace-resolution failure rolls the whole sweep back, and the next tick +// retries idempotently (the bulk-fail queries only re-match tasks still in a +// non-terminal status). Any auto-retry child is a separate downstream write, so +// the emitted event is marked non-retryable. +func (s *TaskService) FailTasksInTxWithEvents(ctx context.Context, bulk func(qtx *db.Queries) ([]db.AgentTaskQueue, error)) ([]db.AgentTaskQueue, error) { + tx, err := s.TxStarter.Begin(ctx) if err != nil { - slog.Warn("domain event: could not resolve task workspace via agent", - "agent_id", util.UUIDToString(agentID), "error", err) - return pgtype.UUID{} + return nil, fmt.Errorf("begin tx: %w", err) } - return agent.WorkspaceID + defer tx.Rollback(ctx) + qtx := s.Queries.WithTx(tx) + + failed, err := bulk(qtx) + if err != nil { + return nil, err + } + for _, t := range failed { + wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) + if !wsID.Valid { + return nil, fmt.Errorf("task.failed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + } + reason := "" + if t.FailureReason.Valid { + reason = t.FailureReason.String + } + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), + domainevent.TaskFailedPayload{ + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + Retryable: false, + ErrorCode: reason, + }) + if _, err := domainevent.Write(ctx, qtx, evt); err != nil { + return nil, err + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return failed, nil } func (s *TaskService) runInTx(ctx context.Context, fn func(*db.Queries) error) error { @@ -4022,17 +4109,32 @@ func (s *TaskService) createAgentComment(ctx context.Context, issueID, agentID p rootComment = &root } } - comment, err := s.Queries.CreateComment(ctx, db.CreateCommentParams{ - IssueID: issueID, - WorkspaceID: issue.WorkspaceID, - AuthorType: "agent", - AuthorID: agentID, - Content: content, - Type: commentType, - ParentID: parentID, - SourceTaskID: sourceTaskID, - }) - if err != nil { + // Transactional outbox (MUL-4332 review point 2): the agent runtime comment + // and its comment.created event commit in one transaction. + var comment db.Comment + if err := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { + created, cErr := qtx.CreateComment(ctx, db.CreateCommentParams{ + IssueID: issueID, + WorkspaceID: issue.WorkspaceID, + AuthorType: "agent", + AuthorID: agentID, + Content: content, + Type: commentType, + ParentID: parentID, + SourceTaskID: sourceTaskID, + }) + if cErr != nil { + return nil, cErr + } + comment = created + return []domainevent.Event{domainevent.CommentCreated(created.WorkspaceID, created.ID, domainevent.AgentActor(agentID), + domainevent.CommentCreatedPayload{ + IssueID: util.UUIDToString(created.IssueID), + AuthorType: "agent", + AuthorID: util.UUIDToString(agentID), + ParentID: util.UUIDToString(parentID), + })}, nil + }); err != nil { return } s.CancelDeferredEscalationsForIssueAgent(ctx, issueID, agentID) diff --git a/server/migrations/193_domain_event.down.sql b/server/migrations/196_domain_event.down.sql similarity index 100% rename from server/migrations/193_domain_event.down.sql rename to server/migrations/196_domain_event.down.sql diff --git a/server/migrations/193_domain_event.up.sql b/server/migrations/196_domain_event.up.sql similarity index 98% rename from server/migrations/193_domain_event.up.sql rename to server/migrations/196_domain_event.up.sql index e8cc9e0c0fd..5c21e2063df 100644 --- a/server/migrations/193_domain_event.up.sql +++ b/server/migrations/196_domain_event.up.sql @@ -20,7 +20,7 @@ -- Workspace DB rules (CLAUDE.md + MUL-4332 §4): NO foreign key, NO cascade — -- every UUID association is validated in the application layer. Secondary and -- unique indexes are added in their own single-statement CONCURRENTLY --- migrations (194–198), never inline, so index builds never take an ACCESS +-- migrations (197–201), never inline, so index builds never take an ACCESS -- EXCLUSIVE lock during deploy. -- Monotonic dispatch / drain boundary. `seq` orders events for stable scanning diff --git a/server/migrations/194_domain_event_seq_unique_index.down.sql b/server/migrations/197_domain_event_seq_unique_index.down.sql similarity index 100% rename from server/migrations/194_domain_event_seq_unique_index.down.sql rename to server/migrations/197_domain_event_seq_unique_index.down.sql diff --git a/server/migrations/194_domain_event_seq_unique_index.up.sql b/server/migrations/197_domain_event_seq_unique_index.up.sql similarity index 100% rename from server/migrations/194_domain_event_seq_unique_index.up.sql rename to server/migrations/197_domain_event_seq_unique_index.up.sql diff --git a/server/migrations/195_domain_event_dispatch_index.down.sql b/server/migrations/198_domain_event_dispatch_index.down.sql similarity index 100% rename from server/migrations/195_domain_event_dispatch_index.down.sql rename to server/migrations/198_domain_event_dispatch_index.down.sql diff --git a/server/migrations/195_domain_event_dispatch_index.up.sql b/server/migrations/198_domain_event_dispatch_index.up.sql similarity index 81% rename from server/migrations/195_domain_event_dispatch_index.up.sql rename to server/migrations/198_domain_event_dispatch_index.up.sql index 557c4b6b556..5ac7e1c6ee5 100644 --- a/server/migrations/195_domain_event_dispatch_index.up.sql +++ b/server/migrations/198_domain_event_dispatch_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 194). Backs the PR3 matcher's +-- Single-statement CONCURRENTLY migration (see 197). Backs the PR3 matcher's -- claim scan for undispatched, now-available events in seq order: -- WHERE dispatch_status = 'pending' AND available_at <= now() ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_dispatch diff --git a/server/migrations/196_domain_event_correlation_index.down.sql b/server/migrations/199_domain_event_correlation_index.down.sql similarity index 100% rename from server/migrations/196_domain_event_correlation_index.down.sql rename to server/migrations/199_domain_event_correlation_index.down.sql diff --git a/server/migrations/196_domain_event_correlation_index.up.sql b/server/migrations/199_domain_event_correlation_index.up.sql similarity index 81% rename from server/migrations/196_domain_event_correlation_index.up.sql rename to server/migrations/199_domain_event_correlation_index.up.sql index fa1314a985b..926d1850621 100644 --- a/server/migrations/196_domain_event_correlation_index.up.sql +++ b/server/migrations/199_domain_event_correlation_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 194). Backs correlation-chain +-- Single-statement CONCURRENTLY migration (see 197). Backs correlation-chain -- reads (GET /api/events?correlation_id=) and loop/depth guardrail lookups: -- WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_correlation diff --git a/server/migrations/197_domain_event_type_index.down.sql b/server/migrations/200_domain_event_type_index.down.sql similarity index 100% rename from server/migrations/197_domain_event_type_index.down.sql rename to server/migrations/200_domain_event_type_index.down.sql diff --git a/server/migrations/197_domain_event_type_index.up.sql b/server/migrations/200_domain_event_type_index.up.sql similarity index 78% rename from server/migrations/197_domain_event_type_index.up.sql rename to server/migrations/200_domain_event_type_index.up.sql index d54d8774880..f1ffe93086e 100644 --- a/server/migrations/197_domain_event_type_index.up.sql +++ b/server/migrations/200_domain_event_type_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 194). Backs per-workspace, +-- Single-statement CONCURRENTLY migration (see 197). Backs per-workspace, -- per-type event scans used by explain/debug tooling: -- WHERE workspace_id = $1 AND type = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_type diff --git a/server/migrations/198_domain_event_subject_index.down.sql b/server/migrations/201_domain_event_subject_index.down.sql similarity index 100% rename from server/migrations/198_domain_event_subject_index.down.sql rename to server/migrations/201_domain_event_subject_index.down.sql diff --git a/server/migrations/198_domain_event_subject_index.up.sql b/server/migrations/201_domain_event_subject_index.up.sql similarity index 81% rename from server/migrations/198_domain_event_subject_index.up.sql rename to server/migrations/201_domain_event_subject_index.up.sql index 03d386916fe..836b95ea72e 100644 --- a/server/migrations/198_domain_event_subject_index.up.sql +++ b/server/migrations/201_domain_event_subject_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 194). Backs "all events about +-- Single-statement CONCURRENTLY migration (see 197). Backs "all events about -- this subject" scans (a given issue / comment / task) for debug + the future -- stage sensor: -- WHERE subject_type = $1 AND subject_id = $2 ORDER BY seq diff --git a/server/pkg/db/generated/domain_event.sql.go b/server/pkg/db/generated/domain_event.sql.go index 95b2662f190..7b49b12a56d 100644 --- a/server/pkg/db/generated/domain_event.sql.go +++ b/server/pkg/db/generated/domain_event.sql.go @@ -116,34 +116,6 @@ func (q *Queries) CreateDomainEvent(ctx context.Context, arg CreateDomainEventPa return i, err } -const deleteDispatchedDomainEventsBefore = `-- name: DeleteDispatchedDomainEventsBefore :execrows -DELETE FROM domain_event -WHERE id IN ( - SELECT de.id FROM domain_event de - WHERE de.dispatch_status = 'dispatched' - AND de.created_at < $1 - ORDER BY de.seq - LIMIT $2 -) -` - -type DeleteDispatchedDomainEventsBeforeParams struct { - CreatedAt pgtype.Timestamptz `json:"created_at"` - Limit int32 `json:"limit"` -} - -// DeleteDispatchedDomainEventsBefore is the retention sweep (MUL-4332 §4.1): -// it only reclaims events already marked 'dispatched' and older than the TTL -// cutoff, in bounded batches so a large backlog never monopolizes the DB. In -// PR1 nothing dispatches events, so this is a no-op until the PR3 matcher runs. -func (q *Queries) DeleteDispatchedDomainEventsBefore(ctx context.Context, arg DeleteDispatchedDomainEventsBeforeParams) (int64, error) { - result, err := q.db.Exec(ctx, deleteDispatchedDomainEventsBefore, arg.CreatedAt, arg.Limit) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil -} - const getDomainEvent = `-- name: GetDomainEvent :one SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at FROM domain_event WHERE id = $1 diff --git a/server/pkg/db/generated/issue.sql.go b/server/pkg/db/generated/issue.sql.go index 2c94c82b140..689af304004 100644 --- a/server/pkg/db/generated/issue.sql.go +++ b/server/pkg/db/generated/issue.sql.go @@ -1159,6 +1159,37 @@ func (q *Queries) LockIssueDuplicateKey(ctx context.Context, dollar_1 string) er return err } +const lockIssueStatusForEvent = `-- name: LockIssueStatusForEvent :one +SELECT status, assignee_type, assignee_id +FROM issue +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE +` + +type LockIssueStatusForEventParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +type LockIssueStatusForEventRow struct { + Status string `json:"status"` + AssigneeType pgtype.Text `json:"assignee_type"` + AssigneeID pgtype.UUID `json:"assignee_id"` +} + +// Row-locks an issue and returns its authoritative status + assignee for +// domain-event emission (MUL-4332 review point 3). Callers read this INSIDE the +// update transaction, immediately before UpdateIssue/UpdateIssueStatus, so the +// event's `from` reflects the truly-current row rather than a snapshot read +// outside the tx — two concurrent transitions then serialize on the lock and +// each records the correct edge instead of both reporting the same stale `from`. +func (q *Queries) LockIssueStatusForEvent(ctx context.Context, arg LockIssueStatusForEventParams) (LockIssueStatusForEventRow, error) { + row := q.db.QueryRow(ctx, lockIssueStatusForEvent, arg.ID, arg.WorkspaceID) + var i LockIssueStatusForEventRow + err := row.Scan(&i.Status, &i.AssigneeType, &i.AssigneeID) + return i, err +} + const markIssueFirstExecuted = `-- name: MarkIssueFirstExecuted :one UPDATE issue SET first_executed_at = now() diff --git a/server/pkg/db/queries/domain_event.sql b/server/pkg/db/queries/domain_event.sql index 0fcfdc986b4..b40677b3546 100644 --- a/server/pkg/db/queries/domain_event.sql +++ b/server/pkg/db/queries/domain_event.sql @@ -40,16 +40,9 @@ SELECT count(*) FROM domain_event WHERE subject_type = $1 AND subject_id = $2; --- DeleteDispatchedDomainEventsBefore is the retention sweep (MUL-4332 §4.1): --- it only reclaims events already marked 'dispatched' and older than the TTL --- cutoff, in bounded batches so a large backlog never monopolizes the DB. In --- PR1 nothing dispatches events, so this is a no-op until the PR3 matcher runs. --- name: DeleteDispatchedDomainEventsBefore :execrows -DELETE FROM domain_event -WHERE id IN ( - SELECT de.id FROM domain_event de - WHERE de.dispatch_status = 'dispatched' - AND de.created_at < $1 - ORDER BY de.seq - LIMIT $2 -); +-- NOTE: the retention/TTL delete is intentionally NOT defined in PR1. The +-- correct predicate is "dispatched AND older than TTL AND every related +-- hook_execution is terminal" (MUL-4332 §4.1/§9), and hook_execution does not +-- exist until PR3. Shipping a weaker "dispatched + TTL" delete now would risk +-- reclaiming still-executing audit sources the moment PR3 enables dispatching +-- (review point 5). The query lands in PR3 with the full terminal predicate. diff --git a/server/pkg/db/queries/issue.sql b/server/pkg/db/queries/issue.sql index 0d603463200..fd0f04cf85c 100644 --- a/server/pkg/db/queries/issue.sql +++ b/server/pkg/db/queries/issue.sql @@ -361,3 +361,15 @@ UPDATE issue SET first_executed_at = now() WHERE id = $1 AND first_executed_at IS NULL RETURNING id, workspace_id, creator_type, creator_id, first_executed_at; + +-- name: LockIssueStatusForEvent :one +-- Row-locks an issue and returns its authoritative status + assignee for +-- domain-event emission (MUL-4332 review point 3). Callers read this INSIDE the +-- update transaction, immediately before UpdateIssue/UpdateIssueStatus, so the +-- event's `from` reflects the truly-current row rather than a snapshot read +-- outside the tx — two concurrent transitions then serialize on the lock and +-- each records the correct edge instead of both reporting the same stale `from`. +SELECT status, assignee_type, assignee_id +FROM issue +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE; From f8e29f7e0317d7627b7de4700c01a82013b0bc25 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 11:53:05 +0800 Subject: [PATCH 03/15] fix(automation): address Elon 2nd-round review on domain event outbox (MUL-4332 PR1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto current main and renumbered the six domain_event migrations to 200–205 (main advanced to 199 with issue_property_icon + agent_task_attribution), restoring the numeric-prefix uniqueness the CI lint enforces. Second-round review must-fixes on PR #5467: 1. Concurrent status-only update no longer rolls back a concurrent writer's assignee (or other nullable field): LockIssueStatusForEvent now locks and returns the full row (LockIssueRowForUpdate); UpdateIssue/BatchUpdateIssues rebuild every untouched bare-narg column from the locked row instead of the pre-tx snapshot, and drive all post-commit side effects (realtime, enqueue, parent-notify) from that authoritative pre-image. 2. Bulk task.failed is now poison-isolated and bounded. Replaced the four condition-scoped fail UPDATEs with bounded candidate SELECTs (FOR UPDATE SKIP LOCKED + LIMIT) + a shared FailAgentTasksByIDs; FailBulkTasksWithEvents resolves each candidate's workspace, fails only the resolvable set with its event atomically, and skips (fail-closed) an unresolvable row without aborting the batch. Offline-runtime task cleanup now runs every tick (sweepOfflineRuntimeTasks), so a rolled-back or leftover orphan is retried instead of stranded. 3. Bulk task.failed events are attributed to SystemActor (sweeper/orphan recovery, not the agent) and carry retryable computed from the shared retryEligible predicate, so the event agrees with the post-commit auto-retry decision instead of hard-coding false. 4. Lock-internal pre-image now drives lock-external side effects: the stuck-issue reset re-checks status==in_progress and no-active-task UNDER the lock (CAS) so a user-completed issue is never reopened; the GitHub merge close suppresses the parent child-done comment / realtime status_changed on a no-op transition. Nit: the domain_event down migration comment now says "Reverses 200". Tests: concurrent reassign-not-rolled-back, poison-row isolation, transient failure next-tick recovery, terminal-vs-reset race (lock-hold), GitHub no-op suppression, and actor/retryable assertions on the bulk fail. Verified against a fresh DB fully migrated to 205 (main + this branch) across domainevent/service/handler/cmd-server, plus full up/down/replay. Co-authored-by: multica-agent --- server/cmd/server/runtime_sweeper.go | 106 ++- server/cmd/server/runtime_sweeper_test.go | 103 +-- server/internal/domainevent/event.go | 10 +- .../handler/domain_event_race_test.go | 134 ++++ server/internal/handler/github.go | 33 +- server/internal/handler/issue.go | 88 ++- server/internal/handler/task_lifecycle.go | 30 +- .../service/domain_event_bulk_fail_test.go | 267 ++++++++ .../service/domain_event_task_failed_test.go | 56 +- server/internal/service/task.go | 121 +++- ...ent.down.sql => 200_domain_event.down.sql} | 2 +- ...n_event.up.sql => 200_domain_event.up.sql} | 2 +- ...01_domain_event_seq_unique_index.down.sql} | 0 ... 201_domain_event_seq_unique_index.up.sql} | 0 ... 202_domain_event_dispatch_index.down.sql} | 0 ...=> 202_domain_event_dispatch_index.up.sql} | 2 +- ...3_domain_event_correlation_index.down.sql} | 0 ...203_domain_event_correlation_index.up.sql} | 2 +- ...l => 204_domain_event_type_index.down.sql} | 0 ...sql => 204_domain_event_type_index.up.sql} | 2 +- ...> 205_domain_event_subject_index.down.sql} | 0 ... => 205_domain_event_subject_index.up.sql} | 2 +- server/pkg/db/generated/agent.sql.go | 619 ++++++++++-------- server/pkg/db/generated/issue.sql.go | 67 +- server/pkg/db/generated/runtime.sql.go | 171 ++--- server/pkg/db/queries/agent.sql | 136 ++-- server/pkg/db/queries/issue.sql | 24 +- server/pkg/db/queries/runtime.sql | 23 +- 28 files changed, 1409 insertions(+), 591 deletions(-) create mode 100644 server/internal/handler/domain_event_race_test.go create mode 100644 server/internal/service/domain_event_bulk_fail_test.go rename server/migrations/{196_domain_event.down.sql => 200_domain_event.down.sql} (75%) rename server/migrations/{196_domain_event.up.sql => 200_domain_event.up.sql} (98%) rename server/migrations/{197_domain_event_seq_unique_index.down.sql => 201_domain_event_seq_unique_index.down.sql} (100%) rename server/migrations/{197_domain_event_seq_unique_index.up.sql => 201_domain_event_seq_unique_index.up.sql} (100%) rename server/migrations/{198_domain_event_dispatch_index.down.sql => 202_domain_event_dispatch_index.down.sql} (100%) rename server/migrations/{198_domain_event_dispatch_index.up.sql => 202_domain_event_dispatch_index.up.sql} (81%) rename server/migrations/{199_domain_event_correlation_index.down.sql => 203_domain_event_correlation_index.down.sql} (100%) rename server/migrations/{199_domain_event_correlation_index.up.sql => 203_domain_event_correlation_index.up.sql} (81%) rename server/migrations/{200_domain_event_type_index.down.sql => 204_domain_event_type_index.down.sql} (100%) rename server/migrations/{200_domain_event_type_index.up.sql => 204_domain_event_type_index.up.sql} (78%) rename server/migrations/{201_domain_event_subject_index.down.sql => 205_domain_event_subject_index.down.sql} (100%) rename server/migrations/{201_domain_event_subject_index.up.sql => 205_domain_event_subject_index.up.sql} (81%) diff --git a/server/cmd/server/runtime_sweeper.go b/server/cmd/server/runtime_sweeper.go index 0261d1c3c57..a3a9cad143c 100644 --- a/server/cmd/server/runtime_sweeper.go +++ b/server/cmd/server/runtime_sweeper.go @@ -76,6 +76,12 @@ const ( chatFinalizeGraceSeconds = 60.0 // chatFinalizeBatchSize caps deferred finalizations per tick. chatFinalizeBatchSize = 100 + // bulkFailBatchSize caps how many tasks a single offline-runtime or + // stale-task sweep fails per tick (MUL-4332 review point 2). Bounding the + // candidate SELECT keeps the FOR UPDATE lock hold short even if a large + // backlog of orphans accumulates; the rest drain on later ticks. 500 mirrors + // queuedExpireBatchSize — far above any realistic per-tick orphan count. + bulkFailBatchSize = 500 ) // runRuntimeSweeper periodically marks runtimes as offline if their @@ -99,6 +105,7 @@ func runRuntimeSweeper(ctx context.Context, queries *db.Queries, liveness handle return case <-ticker.C: sweepStaleRuntimes(ctx, queries, liveness, taskSvc, bus) + sweepOfflineRuntimeTasks(ctx, taskSvc) sweepStaleTasks(ctx, queries, taskSvc, bus) sweepExpiredQueuedTasks(ctx, queries, taskSvc) sweepDeferredChatFinalizations(ctx, queries, taskSvc) @@ -167,17 +174,11 @@ func sweepStaleRuntimes(ctx context.Context, queries *db.Queries, liveness handl slog.Info("runtime sweeper: marked stale runtimes offline", "count", len(staleRows), "workspaces", len(workspaces)) - // Fail orphaned tasks (dispatched/running) whose runtimes just went offline. - // Emit task.failed atomically with the bulk fail (MUL-4332 review point 2). - failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - return qtx.FailTasksForOfflineRuntimes(ctx) - }) - if err != nil { - slog.Warn("runtime sweeper: failed to clean up stale tasks", "error", err) - } else if len(failedTasks) > 0 { - slog.Info("runtime sweeper: failed orphaned tasks", "count", len(failedTasks)) - taskSvc.HandleFailedTasks(ctx, failedTasks) - } + // Orphaned tasks whose runtimes just went offline are reclaimed by + // sweepOfflineRuntimeTasks, which runs every tick against ALL offline + // runtimes (MUL-4332 review point 2) — not only the ones flipped this tick — + // so a rolled-back batch or a runtime that went offline via another path is + // still retried on the next tick rather than stranded here. // Notify frontend clients so they re-fetch runtime list. for wsID := range workspaces { @@ -272,16 +273,26 @@ func gcRuntimes(ctx context.Context, queries *db.Queries, bus *events.Bus) { // edge where a runtime row lingers online-with-stale-heartbeat past the // wall clock (MUL-4107). func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService, bus *events.Bus) { - // Emit task.failed atomically with the bulk fail (MUL-4332 review point 2). - failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - return qtx.FailStaleTasks(ctx, db.FailStaleTasksParams{ - DispatchTimeoutSecs: dispatchTimeoutSeconds, - RunningTimeoutSecs: runningTimeoutSeconds, - // Reuse the runtime stale window so the running-task backstop - // exactly matches what sweepStaleRuntimes considers "not alive". - RuntimeStaleSecs: staleThresholdSeconds, + // Select a bounded batch of stale candidates, then fail the resolvable ones + // with their task.failed events atomically (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectStaleTasksToFail(ctx, db.SelectStaleTasksToFailParams{ + DispatchTimeoutSecs: dispatchTimeoutSeconds, + RunningTimeoutSecs: runningTimeoutSeconds, + // Reuse the runtime stale window so the running-task backstop + // exactly matches what sweepStaleRuntimes considers "not alive". + RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, + }) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "task timed out", Valid: true}, + FailureReason: pgtype.Text{String: "timeout", Valid: true}, + }) }) - }) if err != nil { slog.Warn("task sweeper: failed to clean up stale tasks", "error", err) return @@ -295,6 +306,40 @@ func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service. taskSvc.HandleFailedTasks(ctx, failedTasks) } +// sweepOfflineRuntimeTasks fails orphaned dispatched/running/waiting tasks whose +// runtime is offline, then runs the standard post-fail side effects. Unlike the +// old placement inside sweepStaleRuntimes (which only fired when a runtime was +// flipped offline THIS tick), it runs EVERY tick against ALL offline runtimes +// (MUL-4332 review point 2): a rolled-back batch, a transient event failure, or a +// runtime taken offline via another path is retried on the next tick instead of +// stranding its `waiting_local_directory` orphans. Poison rows are isolated +// inside FailBulkTasksWithEvents so one corrupt row never blocks the rest. +func sweepOfflineRuntimeTasks(ctx context.Context, taskSvc *service.TaskService) { + if taskSvc == nil { + return + } + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectTasksForOfflineRuntimes(ctx, bulkFailBatchSize) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + slog.Warn("runtime sweeper: failed to clean up offline-runtime tasks", "error", err) + return + } + if len(failedTasks) == 0 { + return + } + slog.Info("runtime sweeper: failed orphaned tasks", "count", len(failedTasks)) + taskSvc.HandleFailedTasks(ctx, failedTasks) +} + // sweepExpiredQueuedTasks fails tasks that have been sitting in 'queued' for // longer than the TTL. Companion to the dispatch-time admission gate added // in MUL-1899: that gate prevents new doomed enqueues; this gate drains the @@ -302,13 +347,22 @@ func sweepStaleTasks(ctx context.Context, queries *db.Queries, taskSvc *service. // a task is already queued. Capped to queuedExpireBatchSize per tick so a // big backlog can't monopolise the DB. func sweepExpiredQueuedTasks(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService) { - // Emit task.failed atomically with the bulk expiry (MUL-4332 review point 2). - failedTasks, err := taskSvc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - return qtx.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ - TtlSecs: queuedTTLSeconds, - MaxPerTick: queuedExpireBatchSize, + // Select a bounded batch of TTL-expired queued candidates, then fail the + // resolvable ones with their events atomically (MUL-4332 review point 2). + failedTasks, err := taskSvc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectExpiredQueuedTasks(ctx, db.SelectExpiredQueuedTasksParams{ + TtlSecs: queuedTTLSeconds, + MaxPerTick: queuedExpireBatchSize, + }) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "task expired in queue", Valid: true}, + FailureReason: pgtype.Text{String: "queued_expired", Valid: true}, + }) }) - }) if err != nil { slog.Warn("task sweeper: failed to expire stale queued tasks", "error", err) return diff --git a/server/cmd/server/runtime_sweeper_test.go b/server/cmd/server/runtime_sweeper_test.go index 050bd33e967..402268359ff 100644 --- a/server/cmd/server/runtime_sweeper_test.go +++ b/server/cmd/server/runtime_sweeper_test.go @@ -7,10 +7,55 @@ import ( "testing" "time" + "github.com/jackc/pgx/v5/pgtype" "github.com/multica-ai/multica/server/internal/events" db "github.com/multica-ai/multica/server/pkg/db/generated" ) +// selectAndFailStale / selectAndFailExpiredQueued run the two-step MUL-4332 +// bulk-fail (candidate SELECT then FailAgentTasksByIDs) that replaced the old +// single-statement UPDATE queries, so these predicate tests keep asserting on +// genuinely-failed rows. Poison isolation and workspace resolution are covered +// separately by the FailBulkTasksWithEvents service tests; here we exercise the +// raw SQL predicate + fail. +func selectAndFailStale(t *testing.T, ctx context.Context, queries *db.Queries, p db.SelectStaleTasksToFailParams) []db.AgentTaskQueue { + t.Helper() + candidates, err := queries.SelectStaleTasksToFail(ctx, p) + if err != nil { + t.Fatalf("SelectStaleTasksToFail failed: %v", err) + } + return failCandidatesByIDs(t, ctx, queries, candidates, "task timed out", "timeout") +} + +func selectAndFailExpiredQueued(t *testing.T, ctx context.Context, queries *db.Queries, p db.SelectExpiredQueuedTasksParams) []db.AgentTaskQueue { + t.Helper() + candidates, err := queries.SelectExpiredQueuedTasks(ctx, p) + if err != nil { + t.Fatalf("SelectExpiredQueuedTasks failed: %v", err) + } + return failCandidatesByIDs(t, ctx, queries, candidates, "task expired in queue", "queued_expired") +} + +func failCandidatesByIDs(t *testing.T, ctx context.Context, queries *db.Queries, candidates []db.AgentTaskQueue, errMsg, reason string) []db.AgentTaskQueue { + t.Helper() + if len(candidates) == 0 { + return nil + } + ids := make([]pgtype.UUID, len(candidates)) + for i, c := range candidates { + ids[i] = c.ID + } + failed, err := queries.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: errMsg, Valid: true}, + FailureReason: pgtype.Text{String: reason, Valid: true}, + }) + if err != nil { + t.Fatalf("FailAgentTasksByIDs failed: %v", err) + } + return failed +} + // setupSweeperTestFixture creates an issue and a task in the given status with // timestamps old enough to trigger the sweeper. Returns (issueID, agentID, taskID). func setupSweeperTestFixture(t *testing.T, taskStatus string) (string, string, string) { @@ -175,14 +220,12 @@ func TestSweepStaleTasksBroadcastsWithWorkspaceID(t *testing.T) { }) // Use very short timeouts to trigger the sweep on our test task - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, // 1 second — our task is 3 hours old RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks query failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale task to be failed") } @@ -225,7 +268,7 @@ func TestSweepStaleTasksBroadcastsWithWorkspaceID(t *testing.T) { // Verify DB: task should be failed var status string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) if err != nil { t.Fatalf("failed to query task status: %v", err) } @@ -259,14 +302,12 @@ func TestSweepStaleTasksReconcileAgentStatus(t *testing.T) { }) // Fail stale tasks with short timeout - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale task") } @@ -275,7 +316,7 @@ func TestSweepStaleTasksReconcileAgentStatus(t *testing.T) { // Verify agent status is now "idle" in DB var agentStatus string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent WHERE id = $1`, agentID).Scan(&agentStatus) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent WHERE id = $1`, agentID).Scan(&agentStatus) if err != nil { t.Fatalf("failed to query agent status: %v", err) } @@ -321,16 +362,14 @@ func TestSweepDispatchedStaleTask(t *testing.T) { }) // Fail stale tasks — dispatch timeout of 1 second (our task is 10 minutes old) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 1.0, RunningTimeoutSecs: 9000.0, // RuntimeStaleSecs only affects the running branch — irrelevant for // this dispatched-timeout test, but wired for API consistency. RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } if len(failedTasks) == 0 { t.Fatal("expected at least 1 stale dispatched task") } @@ -339,7 +378,7 @@ func TestSweepDispatchedStaleTask(t *testing.T) { // Verify DB: task should be failed var status string - err = testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) + err := testPool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&status) if err != nil { t.Fatalf("failed to query task: %v", err) } @@ -397,14 +436,12 @@ func TestSweepRunningTaskSkippedWhenRuntimeFresh(t *testing.T) { // Task started_at is 3h ago; RunningTimeoutSecs=1s would kill on wall clock // alone — but the runtime is proving liveness, so the sweeper must skip it. queries := db.New(testPool) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } for _, ft := range failedTasks { if ft.ID.Bytes == parseUUIDBytes(taskID) { @@ -440,14 +477,12 @@ func TestSweepRunningTaskKilledWhenRuntimeStale(t *testing.T) { ageOutAgentRuntime(t, agentID, 10*time.Minute) queries := db.New(testPool) - failedTasks, err := queries.FailStaleTasks(context.Background(), db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, context.Background(), queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } found := false for _, ft := range failedTasks { @@ -531,14 +566,12 @@ func TestSweepResetsInProgressIssueToTodo(t *testing.T) { ageOutAgentRuntime(t, agentID, 10*time.Minute) // Fail the stale task (running timeout of 1 second — our task is 3 hours old). - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, ctx, queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } // Confirm our task was swept. found := false @@ -619,14 +652,12 @@ func TestSweepDoesNotResetIssueAlreadyInReview(t *testing.T) { // Runtime must be stale for the running-task wall clock to fire (MUL-4107). ageOutAgentRuntime(t, agentID, 10*time.Minute) - failedTasks, err := queries.FailStaleTasks(ctx, db.FailStaleTasksParams{ + failedTasks := selectAndFailStale(t, ctx, queries, db.SelectStaleTasksToFailParams{ DispatchTimeoutSecs: 300.0, RunningTimeoutSecs: 1.0, RuntimeStaleSecs: staleThresholdSeconds, + MaxPerTick: bulkFailBatchSize, }) - if err != nil { - t.Fatalf("FailStaleTasks failed: %v", err) - } broadcastFailedTasks(ctx, queries, nil, bus, failedTasks) @@ -706,13 +737,10 @@ func TestExpireStaleQueuedTasks(t *testing.T) { } queries := db.New(testPool) - failed, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ + failed := selectAndFailExpiredQueued(t, ctx, queries, db.SelectExpiredQueuedTasksParams{ TtlSecs: 3600.0, // 1h TTL — old task is 5h, fresh task is 0s MaxPerTick: 100, }) - if err != nil { - t.Fatalf("ExpireStaleQueuedTasks failed: %v", err) - } if len(failed) != 1 { t.Fatalf("expected exactly 1 expired task, got %d", len(failed)) } @@ -802,13 +830,10 @@ func TestExpireStaleQueuedTasksRespectsBatchLimit(t *testing.T) { } queries := db.New(testPool) - failed, err := queries.ExpireStaleQueuedTasks(ctx, db.ExpireStaleQueuedTasksParams{ + failed := selectAndFailExpiredQueued(t, ctx, queries, db.SelectExpiredQueuedTasksParams{ TtlSecs: 3600.0, MaxPerTick: 2, // cap below the backlog }) - if err != nil { - t.Fatalf("ExpireStaleQueuedTasks failed: %v", err) - } if len(failed) != 2 { t.Fatalf("expected batch cap of 2, got %d", len(failed)) } diff --git a/server/internal/domainevent/event.go b/server/internal/domainevent/event.go index c1054c38699..0bf8e86e0e8 100644 --- a/server/internal/domainevent/event.go +++ b/server/internal/domainevent/event.go @@ -166,8 +166,14 @@ func (TaskCompletedPayload) eventType() string { return TypeTaskCompleted } func (TaskCompletedPayload) subjectType() string { return SubjectTask } func (TaskCompletedPayload) schemaVersion() int32 { return 1 } -// TaskFailedPayload — subject is the task. Retryable distinguishes a terminal -// failure from one that spawned an auto-retry child. +// TaskFailedPayload — subject is the task. Retryable reports whether this failure +// is eligible for an automatic retry (an infra-shaped reason still within the +// attempt budget). The single FailTask path creates the retry child in the SAME +// transaction, so there it is exact; the bulk sweeper paths create it immediately +// after commit, so there it means "a retry is expected" — both derive it from the +// shared retryEligible predicate so the event never contradicts the actual retry +// decision. A consumer should read retryable=true as "not yet terminal, a fresh +// attempt is coming" rather than reacting to the failure. type TaskFailedPayload struct { IssueID string `json:"issue_id,omitempty"` AgentID string `json:"agent_id,omitempty"` diff --git a/server/internal/handler/domain_event_race_test.go b/server/internal/handler/domain_event_race_test.go new file mode 100644 index 00000000000..7977d3481be --- /dev/null +++ b/server/internal/handler/domain_event_race_test.go @@ -0,0 +1,134 @@ +package handler + +import ( + "context" + "fmt" + "net/http/httptest" + "sync" + "testing" + + "github.com/multica-ai/multica/server/internal/domainevent" +) + +// A status-only update that races a concurrent reassignment must not roll the +// assignee back to its pre-tx snapshot (MUL-4332 review point 1). UpdateIssue +// writes the nullable columns as bare narg, so before the fix a status-only +// write whose lock landed AFTER the reassignment committed would overwrite the +// assignee with the stale value it read before the tx — silently undoing the +// reassignment, and emitting no issue.assigned event to signal the reversal. +// After the fix every untouched column is rebuilt from the locked row, so the +// reassignment survives regardless of who wins the lock, and its assigned event +// is always present. Looped over fresh issues to exercise both interleavings. +func TestUpdateIssueConcurrentReassignNotRolledBack(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + const iterations = 6 + for i := 0; i < iterations; i++ { + issueID := createTestIssue(t, fmt.Sprintf("reassign-race %s #%d", t.Name(), i), "todo", "none") + func() { + defer func() { + deleteTestIssue(t, issueID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }() + + var wg sync.WaitGroup + wg.Add(2) + // A: status-only update — must NOT touch the assignee. + go func() { + defer wg.Done() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{"status": "in_progress"}) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(httptest.NewRecorder(), req) + }() + // B: reassign to the member — must survive the concurrent status write. + go func() { + defer wg.Done() + req := newRequest("PATCH", "/api/issues/"+issueID, map[string]any{ + "assignee_type": "member", + "assignee_id": testUserID, + }) + req = withURLParam(req, "id", issueID) + testHandler.UpdateIssue(httptest.NewRecorder(), req) + }() + wg.Wait() + + var assigneeType, assigneeID string + if err := testPool.QueryRow(context.Background(), + `SELECT COALESCE(assignee_type, ''), COALESCE(assignee_id::text, '') FROM issue WHERE id = $1`, issueID). + Scan(&assigneeType, &assigneeID); err != nil { + t.Fatalf("iter %d: read issue assignee: %v", i, err) + } + if assigneeType != "member" || assigneeID != testUserID { + t.Fatalf("iter %d: assignee rolled back to (%q, %q), want (member, %s) — a status-only write clobbered the reassignment (review point 1)", + i, assigneeType, assigneeID, testUserID) + } + + var sawAssigned bool + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, issueID) { + if e.Type == domainevent.TypeIssueAssigned && payloadField(t, e.Payload, "to_assignee_id") == testUserID { + sawAssigned = true + } + } + if !sawAssigned { + t.Fatalf("iter %d: missing issue.assigned event for the reassignment", i) + } + }() + } +} + +// advanceIssueToDone (the merged-PR close path) must treat an already-done issue +// as a genuine no-op: the locked pre-image shows no transition, so it emits no +// event AND fires no parent child-done comment / realtime status_changed. A +// stale in_progress snapshot from a duplicate webhook delivery must not re-drive +// those side effects (MUL-4332 review point 4). +func TestAdvanceIssueToDoneNoOpSuppressesDuplicate(t *testing.T) { + if testHandler == nil { + t.Skip("no database connection") + } + ctx := context.Background() + + parentID := createTestIssue(t, "no-op parent "+t.Name(), "in_progress", "none") + childID := createTestIssue(t, "no-op child "+t.Name(), "in_progress", "none") + // Link child to parent and complete it directly (bypassing the handler), so + // no child-done comment has fired yet and the parent starts clean. + if _, err := testPool.Exec(ctx, `UPDATE issue SET parent_issue_id = $1, status = 'done' WHERE id = $2`, parentID, childID); err != nil { + t.Fatalf("link + complete child: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM comment WHERE issue_id IN ($1, $2)`, parentID, childID) + testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id IN ($1, $2)`, parentID, childID) + deleteTestIssue(t, childID) + deleteTestIssue(t, parentID) + }) + + childRow, err := testHandler.Queries.GetIssue(ctx, parseUUID(childID)) + if err != nil { + t.Fatalf("load child: %v", err) + } + // Simulate the stale webhook view: the child is already done in the DB, but + // the caller still holds an in_progress snapshot. + stale := childRow + stale.Status = "in_progress" + + before := issueCommentCount(t, parentID) + testHandler.advanceIssueToDone(ctx, stale, testWorkspaceID) + + if after := issueCommentCount(t, parentID); after != before { + t.Fatalf("parent comment count changed %d→%d — a no-op transition must not re-post the child-done comment (review point 4)", before, after) + } + for _, e := range eventsForSubject(t, domainevent.SubjectIssue, childID) { + if e.Type == domainevent.TypeIssueStatusChanged { + t.Fatalf("a no-op advanceIssueToDone must emit no issue.status_changed event, got %+v", e) + } + } +} + +func issueCommentCount(t *testing.T, issueID string) int { + t.Helper() + var n int + if err := testPool.QueryRow(context.Background(), `SELECT count(*) FROM comment WHERE issue_id = $1`, issueID).Scan(&n); err != nil { + t.Fatalf("count comments: %v", err) + } + return n +} diff --git a/server/internal/handler/github.go b/server/internal/handler/github.go index c4d085e58cd..22ea411d61d 100644 --- a/server/internal/handler/github.go +++ b/server/internal/handler/github.go @@ -1367,16 +1367,18 @@ func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, worksp // atomically with the status flip — but only on a real transition (an // already-done issue produces no event). var updated db.Issue + var before db.Issue if err := domainevent.WriteInTx(ctx, h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { - // Lock + read the authoritative status so the event `from` is correct + // Lock + read the authoritative row so the event `from` is correct // even if another writer moved the issue concurrently (review point 3). - before, err := qtx.LockIssueStatusForEvent(ctx, db.LockIssueStatusForEventParams{ + locked, err := qtx.LockIssueRowForUpdate(ctx, db.LockIssueRowForUpdateParams{ ID: issue.ID, WorkspaceID: issue.WorkspaceID, }) if err != nil { return nil, err } + before = locked u, err := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ ID: issue.ID, Status: "done", @@ -1396,22 +1398,31 @@ func (h *Handler) advanceIssueToDone(ctx context.Context, issue db.Issue, worksp return } + // A merged PR can land for an issue that is already `done` — a duplicate + // webhook, or a concurrent path that won the race. The locked pre-image tells + // us whether THIS call actually transitioned it. On a no-op we already emitted + // no domain event; the parent notification and realtime status_changed must + // be suppressed on the SAME condition, else a no-op re-drives the child-done + // comment / trigger and a spurious "issue updated" (review point 4). + if before.Status == updated.Status { + return + } + // Fire the platform parent-notification path on the same transition the - // HTTP UpdateIssue / BatchUpdateIssues paths use. A merged PR is one of - // the most common ways a sub-issue actually reaches `done`, and skipping - // it here would leave the parent silent for the dominant completion path. - // notifyParentOfChildDone re-checks every guard (prev != done, parent - // exists, parent not terminal), so calling it unconditionally is safe. - h.notifyParentOfChildDone(ctx, issue, updated) + // HTTP UpdateIssue / BatchUpdateIssues paths use, driven by the locked + // pre-image. A merged PR is one of the most common ways a sub-issue actually + // reaches `done`. notifyParentOfChildDone re-checks every guard (prev != done, + // parent exists, parent not terminal). + h.notifyParentOfChildDone(ctx, before, updated) prefix := h.getIssuePrefix(ctx, issue.WorkspaceID) resp := issueToResponse(updated, prefix) h.publish(protocol.EventIssueUpdated, workspaceID, "system", "", map[string]any{ "issue": resp, "status_changed": true, - "prev_status": issue.Status, - "creator_type": issue.CreatorType, - "creator_id": uuidToString(issue.CreatorID), + "prev_status": before.Status, + "creator_type": before.CreatorType, + "creator_id": uuidToString(before.CreatorID), "source": "github_pr_merged", }) } diff --git a/server/internal/handler/issue.go b/server/internal/handler/issue.go index a0e12d96982..7366d1669c9 100644 --- a/server/internal/handler/issue.go +++ b/server/internal/handler/issue.go @@ -2682,6 +2682,14 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { return } } + // Which of the remaining bare-narg columns this request actually targeted. + // An untouched one is rebuilt from the locked row inside the tx (review + // point 1) so a concurrent writer's change is never silently rolled back. + _, touchedStartDate := rawFields["start_date"] + _, touchedDueDate := rawFields["due_date"] + _, touchedParent := rawFields["parent_issue_id"] + _, touchedProject := rawFields["project_id"] + _, touchedStage := rawFields["stage"] attachmentIDs, ok := parseUUIDSliceOrBadRequest(w, req.AttachmentIDs, "attachment_ids") if !ok { @@ -2696,18 +2704,46 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { // Transactional outbox (MUL-4332): commit the issue update and any derived // status_changed / assigned event in one transaction, so a crash can never - // separate the fact from its event. The event `from` is read under a row - // lock INSIDE the tx (not from the pre-tx prevIssue snapshot), so concurrent - // transitions serialize and each records the true edge (review point 3). + // separate the fact from its event. The pre-image is read under a row lock + // INSIDE the tx (not from the pre-tx prevIssue snapshot), so concurrent + // transitions serialize and each records the true edge (review point 3), and + // every untouched nullable column is rebuilt from it (review point 1). var issue db.Issue + var before db.Issue err = domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { - before, err := qtx.LockIssueStatusForEvent(r.Context(), db.LockIssueStatusForEventParams{ + locked, err := qtx.LockIssueRowForUpdate(r.Context(), db.LockIssueRowForUpdateParams{ ID: prevIssue.ID, WorkspaceID: prevIssue.WorkspaceID, }) if err != nil { return nil, err } + before = locked + + // Rebuild every bare-narg column this request did NOT target from the + // locked row, so an unrelated update never clobbers a field a concurrent + // writer just changed (review point 1). assignee_type/id move together + // because validateAssigneePair validated them as a pair above. + if !touchedType && !touchedID { + params.AssigneeType = before.AssigneeType + params.AssigneeID = before.AssigneeID + } + if !touchedStartDate { + params.StartDate = before.StartDate + } + if !touchedDueDate { + params.DueDate = before.DueDate + } + if !touchedParent { + params.ParentIssueID = before.ParentIssueID + } + if !touchedProject { + params.ProjectID = before.ProjectID + } + if !touchedStage { + params.Stage = before.Stage + } + updated, err := qtx.UpdateIssue(r.Context(), params) if err != nil { return nil, err @@ -2739,6 +2775,12 @@ func (h *Handler) UpdateIssue(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "failed to update issue: "+err.Error()) return } + // The locked pre-image is the authoritative prev-state: drive every + // post-commit side effect (realtime publish, enqueue predicate, parent + // notify) from the transition that TRULY happened rather than the pre-tx + // snapshot, which a concurrent writer may have already superseded (review + // point 4). + prevIssue = before if len(attachmentIDs) > 0 { h.linkAttachmentsByIssueIDs(r.Context(), issue.ID, issue.WorkspaceID, attachmentIDs) @@ -3273,6 +3315,13 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { continue } } + // Untouched bare-narg columns are rebuilt from the locked row inside the + // tx (review point 1) so an unrelated field is never rolled back. + _, batchTouchedStartDate := rawUpdates["start_date"] + _, batchTouchedDueDate := rawUpdates["due_date"] + _, batchTouchedParent := rawUpdates["parent_issue_id"] + _, batchTouchedProject := rawUpdates["project_id"] + _, batchTouchedStage := rawUpdates["stage"] actorType, actorID := h.resolveActor(r, userID, workspaceID) batchActorUUID, _ := util.ParseUUID(actorID) @@ -3283,14 +3332,40 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { // is read under a row lock inside the tx so concurrent transitions record // the true edge (review point 3). var issue db.Issue + var before db.Issue if writeErr := domainevent.WriteInTx(r.Context(), h.TxStarter, h.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { - before, err := qtx.LockIssueStatusForEvent(r.Context(), db.LockIssueStatusForEventParams{ + locked, err := qtx.LockIssueRowForUpdate(r.Context(), db.LockIssueRowForUpdateParams{ ID: prevIssue.ID, WorkspaceID: prevIssue.WorkspaceID, }) if err != nil { return nil, err } + before = locked + + // Rebuild every untouched bare-narg column from the locked row so a + // concurrent writer's change is never silently rolled back (review + // point 1). See UpdateIssue for the assignee-pair rationale. + if !batchTouchedType && !batchTouchedID { + params.AssigneeType = before.AssigneeType + params.AssigneeID = before.AssigneeID + } + if !batchTouchedStartDate { + params.StartDate = before.StartDate + } + if !batchTouchedDueDate { + params.DueDate = before.DueDate + } + if !batchTouchedParent { + params.ParentIssueID = before.ParentIssueID + } + if !batchTouchedProject { + params.ProjectID = before.ProjectID + } + if !batchTouchedStage { + params.Stage = before.Stage + } + updatedIssue, err := qtx.UpdateIssue(r.Context(), params) if err != nil { return nil, err @@ -3316,6 +3391,9 @@ func (h *Handler) BatchUpdateIssues(w http.ResponseWriter, r *http.Request) { slog.Warn("batch update issue failed", "issue_id", issueID, "error", writeErr) continue } + // Drive every post-commit side effect from the locked pre-image, the + // transition that truly happened, not the pre-tx snapshot (review point 4). + prevIssue = before prefix := h.getIssuePrefix(r.Context(), issue.WorkspaceID) resp := issueToResponse(issue, prefix) diff --git a/server/internal/handler/task_lifecycle.go b/server/internal/handler/task_lifecycle.go index 3013ea50d89..f4648fcaa09 100644 --- a/server/internal/handler/task_lifecycle.go +++ b/server/internal/handler/task_lifecycle.go @@ -13,6 +13,12 @@ import ( db "github.com/multica-ai/multica/server/pkg/db/generated" ) +// orphanRecoveryBatchSize bounds how many orphaned tasks one recover-orphans +// call fails (MUL-4332 review point 2). A single daemon's in-flight concurrency +// is small, so this is far above any realistic per-restart orphan count; the rare +// overflow is caught by the runtime sweeper's every-tick offline pass. +const orphanRecoveryBatchSize = 500 + // RecoverOrphanedTasks is called by the daemon at startup for each runtime // it owns. It atomically fails any dispatched/running tasks the server still // believes belong to that runtime — those are the tasks the previous daemon @@ -29,12 +35,24 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { return } - // Emit task.failed atomically with the bulk orphan-fail (MUL-4332 review - // point 2), so daemon-driven recovery converges onto the outbox like the - // runtime sweepers. - rows, err := h.TaskService.FailTasksInTxWithEvents(r.Context(), func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - return qtx.RecoverOrphanedTasksForRuntime(r.Context(), parseUUID(runtimeID)) - }) + // Select the runtime's orphaned tasks and fail the resolvable ones with their + // task.failed events atomically (MUL-4332 review point 2), so daemon-driven + // recovery converges onto the outbox like the runtime sweepers and an + // unresolvable poison row cannot block the rest. + rows, err := h.TaskService.FailBulkTasksWithEvents(r.Context(), + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + return qtx.SelectOrphanedTasksForRuntime(r.Context(), db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: parseUUID(runtimeID), + MaxPerTick: orphanRecoveryBatchSize, + }) + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(r.Context(), db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) if err != nil { slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) writeError(w, http.StatusInternalServerError, "recover orphans failed") diff --git a/server/internal/service/domain_event_bulk_fail_test.go b/server/internal/service/domain_event_bulk_fail_test.go new file mode 100644 index 00000000000..5e06e697873 --- /dev/null +++ b/server/internal/service/domain_event_bulk_fail_test.go @@ -0,0 +1,267 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// A single unresolvable "poison" row must not fail-close the whole bulk sweep +// (MUL-4332 review point 2): the resolvable tasks still commit their fact + event +// atomically, and the poison row is skipped (left for ops), not failed. +func TestFailBulkTasksIsolatesPoisonRow(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + seedTask := func() string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + goodID := seedTask() + poisonID := seedTask() + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, []string{goodID, poisonID}) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, []string{goodID, poisonID}) + }) + + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + good, e := qtx.GetAgentTask(ctx, util.MustParseUUID(goodID)) + if e != nil { + return nil, e + } + poison, e := qtx.GetAgentTask(ctx, util.MustParseUUID(poisonID)) + if e != nil { + return nil, e + } + // A real agent_task_queue row always resolves (agent_id is NOT NULL + + // FK), so we present the second candidate as a VIEW with its resolvable + // links stripped — exercising the defensive isolation path directly. The + // DB row (poisonID) is untouched and must be left un-failed. + poison.AgentID = pgtype.UUID{} + poison.IssueID = pgtype.UUID{} + poison.ChatSessionID = pgtype.UUID{} + poison.AutopilotRunID = pgtype.UUID{} + return []db.AgentTaskQueue{good, poison}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("FailBulkTasksWithEvents returned an error — a poison row must not fail the batch: %v", err) + } + + // Only the resolvable task is failed and returned. + if len(failed) != 1 || util.UUIDToString(failed[0].ID) != goodID { + t.Fatalf("expected only the resolvable task returned as failed, got %+v", failed) + } + if s := taskStatusForTest(t, pool, goodID); s != "failed" { + t.Errorf("resolvable task status = %q, want failed", s) + } + if n := subjectEventCount(t, pool, goodID); n != 1 { + t.Errorf("resolvable task events = %d, want 1", n) + } + // The poison row is untouched: not failed, no event. + if s := taskStatusForTest(t, pool, poisonID); s != "running" { + t.Errorf("poison task status = %q, want running (must be left for ops, not failed)", s) + } + if n := subjectEventCount(t, pool, poisonID); n != 0 { + t.Errorf("poison task events = %d, want 0 (no valid event is possible without a workspace)", n) + } +} + +// A transient failure inside the fail transaction must roll the whole batch back +// — no fact is committed without its event — so the task stays selectable and the +// next sweep tick reclaims it (MUL-4332 review point 2). +func TestFailBulkTasksTransientFailureRecoversNextTick(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + sel := func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, e := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if e != nil { + return nil, e + } + return []db.AgentTaskQueue{row}, nil + } + + // Tick 1: the fail step errors (a transient DB blip). The whole batch rolls + // back — the task is NOT failed and no event is written. + boom := errors.New("transient DB blip") + if _, err := svc.FailBulkTasksWithEvents(ctx, sel, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return nil, boom + }); !errors.Is(err, boom) { + t.Fatalf("expected the transient error to surface, got %v", err) + } + if s := taskStatusForTest(t, pool, taskID); s != "running" { + t.Fatalf("after rollback task status = %q, want running (no fact may commit)", s) + } + if n := subjectEventCount(t, pool, taskID); n != 0 { + t.Fatalf("after rollback events = %d, want 0", n) + } + + // Tick 2: the same still-selectable task fails cleanly. + failed, err := svc.FailBulkTasksWithEvents(ctx, sel, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("retry FailBulkTasksWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("retry expected 1 failed task, got %d", len(failed)) + } + if s := taskStatusForTest(t, pool, taskID); s != "failed" { + t.Errorf("after retry task status = %q, want failed", s) + } + if n := subjectEventCount(t, pool, taskID); n != 1 { + t.Errorf("after retry events = %d, want 1", n) + } +} + +// The stuck-issue reset must re-decide UNDER the row lock: if a user moves the +// issue to a terminal status in the window between the pre-tx status read and the +// reset lock, the sweeper must not reopen it (MUL-4332 review point 4). We force +// exactly that interleaving by holding the row lock with an uncommitted move to +// 'done' until the reset is blocked on the lock. +func TestHandleFailedTasksDoesNotReopenUserCompletedIssue(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + if _, err := pool.Exec(ctx, `UPDATE issue SET status = 'in_progress' WHERE id = $1`, issueID); err != nil { + t.Fatalf("set issue in_progress: %v", err) + } + // A non-retryable failure so no auto-retry short-circuits the reset branch. + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, failure_reason, priority, completed_at) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'failed', 'agent_error', 0, now()) + RETURNING id`, agentID, issueID).Scan(&taskID); err != nil { + t.Fatalf("seed failed task: %v", err) + } + failedTask, err := queries.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if err != nil { + t.Fatalf("load failed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, issueID) + }) + + // Holder: lock the issue row and move it to 'done' (uncommitted). A plain + // read (HandleFailedTasks' pre-tx GetIssue) still sees the committed + // 'in_progress', so it enters the reset branch — then blocks on this lock. + locked := make(chan struct{}) + release := make(chan struct{}) + go func() { + tx, e := pool.Begin(context.Background()) + if e != nil { + t.Errorf("holder begin: %v", e) + close(locked) + return + } + defer tx.Rollback(context.Background()) + if _, e := tx.Exec(context.Background(), `SELECT status FROM issue WHERE id = $1 FOR UPDATE`, issueID); e != nil { + t.Errorf("holder lock: %v", e) + close(locked) + return + } + if _, e := tx.Exec(context.Background(), `UPDATE issue SET status = 'done' WHERE id = $1`, issueID); e != nil { + t.Errorf("holder update: %v", e) + close(locked) + return + } + close(locked) + <-release + tx.Commit(context.Background()) + }() + + <-locked + done := make(chan struct{}) + go func() { + svc.HandleFailedTasks(ctx, []db.AgentTaskQueue{failedTask}) + close(done) + }() + // Let HandleFailedTasks read the in_progress snapshot and block on the reset + // lock, then let the user's 'done' commit win the row. + time.Sleep(400 * time.Millisecond) + close(release) + <-done + + if s := issueStatusForTest(t, pool, issueID); s != "done" { + t.Fatalf("issue status = %q, want done — a user-completed issue must not be reopened by the stuck-issue reset (review point 4)", s) + } +} + +func taskStatusForTest(t *testing.T, pool *pgxpool.Pool, taskID string) string { + t.Helper() + var s string + if err := pool.QueryRow(context.Background(), `SELECT status FROM agent_task_queue WHERE id = $1`, taskID).Scan(&s); err != nil { + t.Fatalf("query task status: %v", err) + } + return s +} + +func issueStatusForTest(t *testing.T, pool *pgxpool.Pool, issueID string) string { + t.Helper() + var s string + if err := pool.QueryRow(context.Background(), `SELECT status FROM issue WHERE id = $1`, issueID).Scan(&s); err != nil { + t.Fatalf("query issue status: %v", err) + } + return s +} + +func subjectEventCount(t *testing.T, pool *pgxpool.Pool, subjectID string) int { + t.Helper() + var n int + if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM domain_event WHERE subject_id = $1`, subjectID).Scan(&n); err != nil { + t.Fatalf("count events: %v", err) + } + return n +} diff --git a/server/internal/service/domain_event_task_failed_test.go b/server/internal/service/domain_event_task_failed_test.go index bdbc23a5778..1e9ebe7e3dc 100644 --- a/server/internal/service/domain_event_task_failed_test.go +++ b/server/internal/service/domain_event_task_failed_test.go @@ -12,11 +12,13 @@ import ( db "github.com/multica-ai/multica/server/pkg/db/generated" ) -// FailTasksInTxWithEvents is the shared mechanism behind every bulk task.failed -// path (the three runtime sweepers + daemon orphan recovery). Failing a task -// through it must persist a task.failed domain event atomically with the fail -// (MUL-4332 review point 2), stamped with the resolved workspace and issue. -func TestFailTasksInTxWithEventsEmitsTaskFailed(t *testing.T) { +// FailBulkTasksWithEvents is the shared mechanism behind every bulk task.failed +// path (the runtime sweepers + daemon orphan recovery). Failing a task through it +// must persist a task.failed domain event atomically with the fail (MUL-4332 +// review point 2), stamped with the resolved workspace and issue, attributed to +// the platform (SystemActor, not the agent) and carrying a retryable flag that +// agrees with the auto-retry decision (review point 3). +func TestFailBulkTasksWithEventsEmitsTaskFailed(t *testing.T) { pool := newTaskClaimRacePool(t) // skips if no DB ctx := context.Background() queries := db.New(pool) @@ -37,34 +39,48 @@ func TestFailTasksInTxWithEventsEmitsTaskFailed(t *testing.T) { pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) }) - failed, err := svc.FailTasksInTxWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - row, ferr := qtx.FailAgentTask(ctx, db.FailAgentTaskParams{ - ID: util.MustParseUUID(taskID), - Error: pgtype.Text{String: "runtime went offline", Valid: true}, - FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, gerr := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if gerr != nil { + return nil, gerr + } + return []db.AgentTaskQueue{row}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) }) - if ferr != nil { - return nil, ferr - } - return []db.AgentTaskQueue{row}, nil - }) if err != nil { - t.Fatalf("FailTasksInTxWithEvents: %v", err) + t.Fatalf("FailBulkTasksWithEvents: %v", err) } if len(failed) != 1 { t.Fatalf("expected 1 failed task, got %d", len(failed)) } - // Exactly one task.failed event for the task, carrying the issue + error. - var evtType, payload string + // Exactly one task.failed event for the task, carrying the issue + error, + // attributed to the platform, with a retryable flag matching the shared + // retryEligible predicate. + var evtType, actorType, payload string + var retryable bool if err := pool.QueryRow(ctx, - `SELECT type, payload::text FROM domain_event WHERE subject_type = 'task' AND subject_id = $1`, - taskID).Scan(&evtType, &payload); err != nil { + `SELECT type, actor_type, (payload->>'retryable')::bool, payload::text + FROM domain_event WHERE subject_type = 'task' AND subject_id = $1`, + taskID).Scan(&evtType, &actorType, &retryable, &payload); err != nil { t.Fatalf("expected a task.failed domain event: %v", err) } if evtType != "task.failed" { t.Errorf("type = %q, want task.failed", evtType) } + if actorType != "system" { + t.Errorf("actor_type = %q, want system (a sweeper fail is platform-driven, not the agent's action)", actorType) + } + if want := retryEligible("runtime_offline", failed[0]); retryable != want { + t.Errorf("retryable = %v, want %v (event must agree with the auto-retry decision)", retryable, want) + } if !strings.Contains(payload, issueID) { t.Errorf("payload %s should carry issue_id %s", payload, issueID) } diff --git a/server/internal/service/task.go b/server/internal/service/task.go index ca34469fff4..d451e27df9f 100644 --- a/server/internal/service/task.go +++ b/server/internal/service/task.go @@ -3471,18 +3471,34 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas ) } else if !hasActive { var updatedIssue db.Issue + var didReset bool // Transactional outbox (MUL-4332): the stuck-issue reset bypasses the // HTTP UpdateIssue path, so emit issue.status_changed atomically here. updateErr := domainevent.WriteInTx(ctx, s.TxStarter, s.Queries, func(qtx *db.Queries) ([]domainevent.Event, error) { - // Lock + read the true current status inside the tx so the event - // `from` is correct under concurrent transitions (review point 3). - before, bErr := qtx.LockIssueStatusForEvent(ctx, db.LockIssueStatusForEventParams{ + // Re-decide the reset UNDER the row lock (review point 4). The + // in_progress + no-active-task check that gated this branch was read + // outside the tx, so within the window a user could have moved the + // issue to done, or a fresh task could have been claimed. Only a + // still-in_progress issue with still no active task is reset — a + // user-completed issue must never be silently reopened. `from` then + // always reflects the true edge (also review point 3). + before, bErr := qtx.LockIssueRowForUpdate(ctx, db.LockIssueRowForUpdateParams{ ID: t.IssueID, WorkspaceID: issue.WorkspaceID, }) if bErr != nil { return nil, bErr } + if before.Status != "in_progress" { + return nil, nil + } + stillActive, aErr := qtx.HasActiveTaskForIssue(ctx, t.IssueID) + if aErr != nil { + return nil, aErr + } + if stillActive { + return nil, nil + } u, uErr := qtx.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{ ID: t.IssueID, Status: "todo", @@ -3492,9 +3508,7 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas return nil, uErr } updatedIssue = u - if before.Status == u.Status { - return nil, nil - } + didReset = true return []domainevent.Event{domainevent.IssueStatusChanged(u.WorkspaceID, u.ID, domainevent.SystemActor(), domainevent.IssueStatusChangedPayload{From: before.Status, To: u.Status})}, nil }) @@ -3503,13 +3517,14 @@ func (s *TaskService) HandleFailedTasks(ctx context.Context, tasks []db.AgentTas "issue_id", issueKey, "error", updateErr, ) - } else { + } else if didReset { // This direct reset bypasses the HTTP UpdateIssue // handler that normally emits issue:updated, so emit // it here too. Without it the board / status-filter // caches keep showing the issue as in_progress until - // the next write touches it (#4648 / MUL-3782). - s.broadcastIssueUpdated(updatedIssue, issue.Status) + // the next write touches it (#4648 / MUL-3782). The reset + // only fires from in_progress, so that is the true prev. + s.broadcastIssueUpdated(updatedIssue, "in_progress") } } } @@ -3587,16 +3602,37 @@ func (s *TaskService) resolveTaskWorkspaceForEvent(ctx context.Context, qtx *db. return pgtype.UUID{} } -// FailTasksInTxWithEvents runs a bulk-fail query inside a transaction and emits -// a task.failed domain event for each returned task atomically with the bulk -// UPDATE (MUL-4332 review point 2). The runtime sweepers and daemon -// orphan-recovery all funnel through this, so no bulk terminal path commits a -// fail without its event. It is all-or-nothing per batch: an event-write or -// workspace-resolution failure rolls the whole sweep back, and the next tick -// retries idempotently (the bulk-fail queries only re-match tasks still in a -// non-terminal status). Any auto-retry child is a separate downstream write, so -// the emitted event is marked non-retryable. -func (s *TaskService) FailTasksInTxWithEvents(ctx context.Context, bulk func(qtx *db.Queries) ([]db.AgentTaskQueue, error)) ([]db.AgentTaskQueue, error) { +// FailBulkTasksWithEvents is the poison-isolated bulk fail path (MUL-4332 review +// points 2 & 3). Rather than one condition-scoped UPDATE that fails every match +// in a single statement — where one corrupt row rolls back the whole batch and +// an unbounded match can lock a huge span — it runs, in ONE transaction: +// +// 1. selectFn row-locks a BOUNDED candidate batch (FOR UPDATE SKIP LOCKED + +// LIMIT), so the lock hold is short and we never contend with a daemon's +// claim path; +// 2. each candidate's workspace is resolved on the same tx. A task whose +// workspace is genuinely unresolvable (corrupt / orphaned historical row) +// is EXCLUDED and logged — it can never have a valid event, so failing it +// would strand a fact without one; skipping it fail-closed keeps the healthy +// tasks in the batch unblocked (review point 2); +// 3. failFn fails only the resolvable id set; +// 4. one task.failed event is written per failed row, atomically with the fail. +// +// The actor is the platform (SystemActor): these are sweeper / orphan-recovery +// paths, not an agent action — the agent is already carried in the payload +// (review point 3). retryable is computed with the SAME retryEligible predicate +// HandleFailedTasks uses to decide the post-commit auto-retry, so the event and +// the retry decision agree instead of the event hard-coding false while a retry +// child appears anyway. +// +// A transient event/commit failure rolls back only the CURRENT batch (no fact is +// ever committed without its event), so the next sweep tick simply re-selects and +// retries — every caller runs on a fixed cadence. +func (s *TaskService) FailBulkTasksWithEvents( + ctx context.Context, + selectFn func(qtx *db.Queries) ([]db.AgentTaskQueue, error), + failFn func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error), +) ([]db.AgentTaskQueue, error) { tx, err := s.TxStarter.Begin(ctx) if err != nil { return nil, fmt.Errorf("begin tx: %w", err) @@ -3604,24 +3640,61 @@ func (s *TaskService) FailTasksInTxWithEvents(ctx context.Context, bulk func(qtx defer tx.Rollback(ctx) qtx := s.Queries.WithTx(tx) - failed, err := bulk(qtx) + candidates, err := selectFn(qtx) if err != nil { return nil, err } - for _, t := range failed { + if len(candidates) == 0 { + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return nil, nil + } + + ids := make([]pgtype.UUID, 0, len(candidates)) + wsByID := make(map[string]pgtype.UUID, len(candidates)) + for _, t := range candidates { wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) if !wsID.Valid { - return nil, fmt.Errorf("task.failed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) + // Fail-closed but ISOLATED: a task with no resolvable workspace can + // have no valid event, so skip it (leaving it for ops to reap) instead + // of aborting the batch and stranding every healthy task with it. + slog.Warn("bulk fail: skipping task with unresolvable workspace", + "task_id", util.UUIDToString(t.ID), + "agent_id", util.UUIDToString(t.AgentID)) + continue + } + ids = append(ids, t.ID) + wsByID[util.UUIDToString(t.ID)] = wsID + } + if len(ids) == 0 { + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return nil, nil + } + + failed, err := failFn(qtx, ids) + if err != nil { + return nil, err + } + for _, t := range failed { + wsID, ok := wsByID[util.UUIDToString(t.ID)] + if !ok || !wsID.Valid { + // Defensive: failFn returned a row we never resolved. Impossible in one + // tx (ids came from wsByID), but fail-closed rather than emit an event + // with no workspace. + return nil, fmt.Errorf("task.failed event: unresolved workspace for task %s", util.UUIDToString(t.ID)) } reason := "" if t.FailureReason.Valid { reason = t.FailureReason.String } - evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), + evt := domainevent.TaskFailed(wsID, t.ID, domainevent.SystemActor(), domainevent.TaskFailedPayload{ IssueID: util.UUIDToString(t.IssueID), AgentID: util.UUIDToString(t.AgentID), - Retryable: false, + Retryable: retryEligible(reason, t), ErrorCode: reason, }) if _, err := domainevent.Write(ctx, qtx, evt); err != nil { diff --git a/server/migrations/196_domain_event.down.sql b/server/migrations/200_domain_event.down.sql similarity index 75% rename from server/migrations/196_domain_event.down.sql rename to server/migrations/200_domain_event.down.sql index e1d89df7ed7..7b672f13edb 100644 --- a/server/migrations/196_domain_event.down.sql +++ b/server/migrations/200_domain_event.down.sql @@ -1,4 +1,4 @@ --- Reverses 193_domain_event.up.sql. Dropping the table also drops the +-- Reverses 200_domain_event.up.sql. Dropping the table also drops the -- sequence it OWNS; the explicit DROP SEQUENCE is a defensive no-op in case -- the ownership link is ever severed. DROP TABLE IF EXISTS domain_event; diff --git a/server/migrations/196_domain_event.up.sql b/server/migrations/200_domain_event.up.sql similarity index 98% rename from server/migrations/196_domain_event.up.sql rename to server/migrations/200_domain_event.up.sql index 5c21e2063df..3dbf9d3a3a1 100644 --- a/server/migrations/196_domain_event.up.sql +++ b/server/migrations/200_domain_event.up.sql @@ -20,7 +20,7 @@ -- Workspace DB rules (CLAUDE.md + MUL-4332 §4): NO foreign key, NO cascade — -- every UUID association is validated in the application layer. Secondary and -- unique indexes are added in their own single-statement CONCURRENTLY --- migrations (197–201), never inline, so index builds never take an ACCESS +-- migrations (201–205), never inline, so index builds never take an ACCESS -- EXCLUSIVE lock during deploy. -- Monotonic dispatch / drain boundary. `seq` orders events for stable scanning diff --git a/server/migrations/197_domain_event_seq_unique_index.down.sql b/server/migrations/201_domain_event_seq_unique_index.down.sql similarity index 100% rename from server/migrations/197_domain_event_seq_unique_index.down.sql rename to server/migrations/201_domain_event_seq_unique_index.down.sql diff --git a/server/migrations/197_domain_event_seq_unique_index.up.sql b/server/migrations/201_domain_event_seq_unique_index.up.sql similarity index 100% rename from server/migrations/197_domain_event_seq_unique_index.up.sql rename to server/migrations/201_domain_event_seq_unique_index.up.sql diff --git a/server/migrations/198_domain_event_dispatch_index.down.sql b/server/migrations/202_domain_event_dispatch_index.down.sql similarity index 100% rename from server/migrations/198_domain_event_dispatch_index.down.sql rename to server/migrations/202_domain_event_dispatch_index.down.sql diff --git a/server/migrations/198_domain_event_dispatch_index.up.sql b/server/migrations/202_domain_event_dispatch_index.up.sql similarity index 81% rename from server/migrations/198_domain_event_dispatch_index.up.sql rename to server/migrations/202_domain_event_dispatch_index.up.sql index 5ac7e1c6ee5..45738132b2a 100644 --- a/server/migrations/198_domain_event_dispatch_index.up.sql +++ b/server/migrations/202_domain_event_dispatch_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 197). Backs the PR3 matcher's +-- Single-statement CONCURRENTLY migration (see 201). Backs the PR3 matcher's -- claim scan for undispatched, now-available events in seq order: -- WHERE dispatch_status = 'pending' AND available_at <= now() ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_dispatch diff --git a/server/migrations/199_domain_event_correlation_index.down.sql b/server/migrations/203_domain_event_correlation_index.down.sql similarity index 100% rename from server/migrations/199_domain_event_correlation_index.down.sql rename to server/migrations/203_domain_event_correlation_index.down.sql diff --git a/server/migrations/199_domain_event_correlation_index.up.sql b/server/migrations/203_domain_event_correlation_index.up.sql similarity index 81% rename from server/migrations/199_domain_event_correlation_index.up.sql rename to server/migrations/203_domain_event_correlation_index.up.sql index 926d1850621..525149ebcc0 100644 --- a/server/migrations/199_domain_event_correlation_index.up.sql +++ b/server/migrations/203_domain_event_correlation_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 197). Backs correlation-chain +-- Single-statement CONCURRENTLY migration (see 201). Backs correlation-chain -- reads (GET /api/events?correlation_id=) and loop/depth guardrail lookups: -- WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_correlation diff --git a/server/migrations/200_domain_event_type_index.down.sql b/server/migrations/204_domain_event_type_index.down.sql similarity index 100% rename from server/migrations/200_domain_event_type_index.down.sql rename to server/migrations/204_domain_event_type_index.down.sql diff --git a/server/migrations/200_domain_event_type_index.up.sql b/server/migrations/204_domain_event_type_index.up.sql similarity index 78% rename from server/migrations/200_domain_event_type_index.up.sql rename to server/migrations/204_domain_event_type_index.up.sql index f1ffe93086e..6deea6519e8 100644 --- a/server/migrations/200_domain_event_type_index.up.sql +++ b/server/migrations/204_domain_event_type_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 197). Backs per-workspace, +-- Single-statement CONCURRENTLY migration (see 201). Backs per-workspace, -- per-type event scans used by explain/debug tooling: -- WHERE workspace_id = $1 AND type = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_type diff --git a/server/migrations/201_domain_event_subject_index.down.sql b/server/migrations/205_domain_event_subject_index.down.sql similarity index 100% rename from server/migrations/201_domain_event_subject_index.down.sql rename to server/migrations/205_domain_event_subject_index.down.sql diff --git a/server/migrations/201_domain_event_subject_index.up.sql b/server/migrations/205_domain_event_subject_index.up.sql similarity index 81% rename from server/migrations/201_domain_event_subject_index.up.sql rename to server/migrations/205_domain_event_subject_index.up.sql index 836b95ea72e..b4f36da6f57 100644 --- a/server/migrations/201_domain_event_subject_index.up.sql +++ b/server/migrations/205_domain_event_subject_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 197). Backs "all events about +-- Single-statement CONCURRENTLY migration (see 201). Backs "all events about -- this subject" scans (a given issue / comment / task) for debug + the future -- stage sensor: -- WHERE subject_type = $1 AND subject_id = $2 ORDER BY seq diff --git a/server/pkg/db/generated/agent.sql.go b/server/pkg/db/generated/agent.sql.go index d61bf0c163f..a3e50405f25 100644 --- a/server/pkg/db/generated/agent.sql.go +++ b/server/pkg/db/generated/agent.sql.go @@ -1925,124 +1925,6 @@ func (q *Queries) DeleteSystemAgentByID(ctx context.Context, id pgtype.UUID) err return err } -const expireStaleQueuedTasks = `-- name: ExpireStaleQueuedTasks :many -WITH victims AS ( - SELECT id FROM agent_task_queue - WHERE status = 'queued' - AND created_at < now() - make_interval(secs => $1::double precision) - ORDER BY created_at ASC - LIMIT $2::int - FOR UPDATE SKIP LOCKED -) -UPDATE agent_task_queue t -SET status = 'failed', - completed_at = now(), - error = 'task expired in queue', - failure_reason = 'queued_expired', - prepare_lease_expires_at = NULL -FROM victims v -WHERE t.id = v.id - AND t.status = 'queued' - AND t.created_at < now() - make_interval(secs => $1::double precision) -RETURNING t.id, t.agent_id, t.issue_id, t.status, t.priority, t.dispatched_at, t.started_at, t.completed_at, t.result, t.error, t.created_at, t.context, t.runtime_id, t.session_id, t.work_dir, t.trigger_comment_id, t.chat_session_id, t.autopilot_run_id, t.attempt, t.max_attempts, t.parent_task_id, t.failure_reason, t.trigger_summary, t.force_fresh_session, t.is_leader_task, t.wait_reason, t.initiator_user_id, t.handoff_note, t.prepare_lease_expires_at, t.squad_id, t.runtime_mcp_overlay, t.escalation_for_task_id, t.fire_at, t.originator_user_id, t.runtime_connected_apps, t.coalesced_comment_ids, t.delivered_comment_ids, t.chat_input_task_id, t.chat_finalize_deferred_at, t.originator_source, t.delegated_from_task_id, t.retry_of_task_id, t.rerun_of_task_id, t.rule_version_id, t.trigger_evidence_kind, t.trigger_evidence_ref_id, t.accountable_user_id -` - -type ExpireStaleQueuedTasksParams struct { - TtlSecs float64 `json:"ttl_secs"` - MaxPerTick int32 `json:"max_per_tick"` -} - -// Fails tasks that have been sitting in 'queued' for longer than the TTL. -// This is the cleanup arm of the MUL-1899 "queued backlog" fix: even with the -// new dispatch-time admission gate that refuses to enqueue when the runtime -// is offline, we still need to drain the historical 87k+ doomed rows and -// handle edge cases where a runtime goes offline AFTER a task is already -// queued (the admission check protects new enqueues, not in-flight queue -// depth). -// -// Concurrency safety: the daemon's claim path may race with this sweeper to -// transition the same row out of 'queued'. We protect against that two -// ways: -// 1. The CTE selects victims with FOR UPDATE SKIP LOCKED so a row that is -// currently being claimed (or otherwise locked) is skipped — no lock -// contention with the dispatch path, and we won't queue up behind it. -// 2. The outer UPDATE re-checks status='queued' AND the TTL predicate at -// apply time. If a daemon claimed the row between selection and update -// (e.g. lock released after the claim transaction commits), the row is -// already 'dispatched'/'running' and the WHERE clause filters it out -// so we cannot clobber an in-flight task. -// -// Capped via LIMIT inside the CTE so a single sweep tick cannot monopolise -// the DB when the backlog is large — the sweeper drains the rest on -// subsequent ticks. -func (q *Queries) ExpireStaleQueuedTasks(ctx context.Context, arg ExpireStaleQueuedTasksParams) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, expireStaleQueuedTasks, arg.TtlSecs, arg.MaxPerTick) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const extendAgentTaskPrepareLease = `-- name: ExtendAgentTaskPrepareLease :one UPDATE agent_task_queue SET prepare_lease_expires_at = now() + make_interval(secs => $3::double precision) @@ -2209,74 +2091,36 @@ func (q *Queries) FailAgentTask(ctx context.Context, arg FailAgentTaskParams) (A return i, err } -const failStaleTasks = `-- name: FailStaleTasks :many +const failAgentTasksByIDs = `-- name: FailAgentTasksByIDs :many UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'task timed out', - failure_reason = 'timeout', +SET status = 'failed', + completed_at = now(), + error = $1, + failure_reason = $2, + wait_reason = NULL, prepare_lease_expires_at = NULL -WHERE ( - status = 'dispatched' - AND dispatched_at < now() - make_interval(secs => $1::double precision) - AND (prepare_lease_expires_at IS NULL OR prepare_lease_expires_at < now()) - ) - OR ( - status = 'running' - AND started_at < now() - make_interval(secs => $2::double precision) - AND NOT EXISTS ( - SELECT 1 FROM agent_runtime r - WHERE r.id = agent_task_queue.runtime_id - AND r.status = 'online' - AND r.last_seen_at >= now() - make_interval(secs => $3::double precision) - ) - ) +WHERE id = ANY($3::uuid[]) + AND status IN ('queued', 'dispatched', 'running', 'waiting_local_directory') RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id ` -type FailStaleTasksParams struct { - DispatchTimeoutSecs float64 `json:"dispatch_timeout_secs"` - RunningTimeoutSecs float64 `json:"running_timeout_secs"` - RuntimeStaleSecs float64 `json:"runtime_stale_secs"` +type FailAgentTasksByIDsParams struct { + Error pgtype.Text `json:"error"` + FailureReason pgtype.Text `json:"failure_reason"` + Ids []pgtype.UUID `json:"ids"` } -// Fails tasks stuck in dispatched/running beyond the given thresholds. -// -// Each branch pairs a wall-clock deadline with a task-appropriate liveness -// signal, so the sweeper only kills tasks whose owning daemon is no longer -// proving it is alive: -// -// - Dispatched: `prepare_lease_expires_at` is refreshed every 15s by the -// daemon between claim and StartTask (see startTaskPrepareLeaseExtender). -// A live lease excludes the row. -// -// - Running: no per-task lease is renewed once StartTask fires, so we key -// off the daemon-wide heartbeat instead — `agent_runtime.last_seen_at`, -// which the daemon bumps every ~15s while it is up. A running task whose -// runtime is `online` AND whose `last_seen_at` is within -// @runtime_stale_secs is treated as alive and is NOT killed by this -// wall-clock backstop, even after `started_at` exceeds the running -// timeout. This is what lets healthy multi-hour research / training runs -// survive on self-hosted deployments (MUL-4107): the daemon side is -// bounded only by inactivity watchdogs (idle / per-tool), so the -// server-side wall clock must not shadow that with a coarser cap. -// -// The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` -// (which mixes DB `last_seen_at` with the Redis LivenessStore and calls -// `FailTasksForOfflineRuntimes` in the same tick). The wall-clock branch -// here is a defensive backstop for pathological cases where a runtime row -// somehow retains status='online' with a stale DB heartbeat for longer than -// the wall clock allows. -// -// runtime_id IS NULL: a running row with no runtime is by definition not -// proving liveness, so the wall clock is allowed to fire — same shape as -// the legacy pure-wall-clock behavior for that (rare / historical) case. -// -// waiting_local_directory rows are intentionally excluded: the daemon owns -// the wait (with its own ctx-driven timeout) and a legitimate queue ahead -// of this task can exceed the dispatch / running timeouts without being -// "stuck". If the daemon dies, RecoverOrphanedTasksForRuntime reclaims -// those rows at restart. -func (q *Queries) FailStaleTasks(ctx context.Context, arg FailStaleTasksParams) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, failStaleTasks, arg.DispatchTimeoutSecs, arg.RunningTimeoutSecs, arg.RuntimeStaleSecs) +// Fails a specific, already-resolved set of tasks by id — the terminal half of +// the MUL-4332 poison-isolated bulk fail (review point 2). The caller has just +// selected these ids with SelectStaleTasksToFail / SelectExpiredQueuedTasks / +// SelectTasksForOfflineRuntimes / SelectOrphanedTasksForRuntime FOR UPDATE in +// the SAME transaction, so the rows are locked and cannot have transitioned; +// the status guard is a defensive backstop that keeps this idempotent if the id +// set is ever reused. @error / @failure_reason are supplied per sweeper. Both +// wait_reason and prepare_lease_expires_at are cleared (clearing a NULL is a +// no-op) so this one query serves every bulk-fail caller. +func (q *Queries) FailAgentTasksByIDs(ctx context.Context, arg FailAgentTasksByIDsParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, failAgentTasksByIDs, arg.Error, arg.FailureReason, arg.Ids) if err != nil { return nil, err } @@ -4455,92 +4299,6 @@ func (q *Queries) ReclaimStaleDispatchedTasksForRuntimes(ctx context.Context, ar return items, nil } -const recoverOrphanedTasksForRuntime = `-- name: RecoverOrphanedTasksForRuntime :many -UPDATE agent_task_queue -SET status = 'failed', - completed_at = now(), - error = 'daemon restarted while task was in flight', - failure_reason = 'runtime_recovery', - wait_reason = NULL, - prepare_lease_expires_at = NULL -WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') -RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id -` - -// Called by the daemon at startup. Atomically fails any dispatched/running/ -// waiting_local_directory task that the prior incarnation of this runtime -// owned but did not finalize. Returns the failed rows so callers can hand -// them to the auto-retry path. waiting_local_directory rows are included -// because the daemon holding the path lock is the same process that just -// died — without us, the row would sit waiting forever. -func (q *Queries) RecoverOrphanedTasksForRuntime(ctx context.Context, runtimeID pgtype.UUID) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, recoverOrphanedTasksForRuntime, runtimeID) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const refreshAgentStatusFromTasks = `-- name: RefreshAgentStatusFromTasks :one UPDATE agent AS a SET status = CASE WHEN EXISTS ( @@ -4706,6 +4464,337 @@ func (q *Queries) RestoreAgent(ctx context.Context, id pgtype.UUID) (Agent, erro return i, err } +const selectExpiredQueuedTasks = `-- name: SelectExpiredQueuedTasks :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE status = 'queued' + AND created_at < now() - make_interval(secs => $1::double precision) +ORDER BY created_at ASC +LIMIT $2::int +FOR UPDATE SKIP LOCKED +` + +type SelectExpiredQueuedTasksParams struct { + TtlSecs float64 `json:"ttl_secs"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Selects (and row-locks) up to @max_per_tick tasks sitting in 'queued' past the +// TTL — the cleanup arm of the MUL-1899 "queued backlog" fix (drains the +// historical 87k+ doomed rows and catches the case where a runtime goes offline +// AFTER a task is already queued). +// +// Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +// caller resolves each task's workspace, fails only the resolvable set via +// FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +// FOR UPDATE SKIP LOCKED skips rows a daemon is currently claiming, so we never +// contend with the dispatch path or clobber an in-flight task; because the fail +// runs in this same transaction the locked rows cannot transition out of 'queued' +// underneath us. The LIMIT bounds the lock hold; the rest drain on later ticks. +func (q *Queries) SelectExpiredQueuedTasks(ctx context.Context, arg SelectExpiredQueuedTasksParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectExpiredQueuedTasks, arg.TtlSecs, arg.MaxPerTick) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectOrphanedTasksForRuntime = `-- name: SelectOrphanedTasksForRuntime :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') +ORDER BY created_at ASC +LIMIT $2::int +FOR UPDATE SKIP LOCKED +` + +type SelectOrphanedTasksForRuntimeParams struct { + RuntimeID pgtype.UUID `json:"runtime_id"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Called by the daemon at startup. Selects (and row-locks) up to @max_per_tick +// dispatched/running/waiting_local_directory tasks that the prior incarnation of +// this runtime owned but did not finalize. waiting_local_directory rows are +// included because the daemon holding the path lock is the same process that +// just died — without us, the row would sit waiting forever. +// +// Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +// caller resolves each task's workspace, fails only the resolvable set via +// FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +// FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's +// own claim path; the LIMIT bounds the lock hold. +func (q *Queries) SelectOrphanedTasksForRuntime(ctx context.Context, arg SelectOrphanedTasksForRuntimeParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectOrphanedTasksForRuntime, arg.RuntimeID, arg.MaxPerTick) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectStaleTasksToFail = `-- name: SelectStaleTasksToFail :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE ( + status = 'dispatched' + AND dispatched_at < now() - make_interval(secs => $1::double precision) + AND (prepare_lease_expires_at IS NULL OR prepare_lease_expires_at < now()) + ) + OR ( + status = 'running' + AND started_at < now() - make_interval(secs => $2::double precision) + AND NOT EXISTS ( + SELECT 1 FROM agent_runtime r + WHERE r.id = agent_task_queue.runtime_id + AND r.status = 'online' + AND r.last_seen_at >= now() - make_interval(secs => $3::double precision) + ) + ) +ORDER BY started_at ASC NULLS FIRST, dispatched_at ASC NULLS FIRST +LIMIT $4::int +FOR UPDATE SKIP LOCKED +` + +type SelectStaleTasksToFailParams struct { + DispatchTimeoutSecs float64 `json:"dispatch_timeout_secs"` + RunningTimeoutSecs float64 `json:"running_timeout_secs"` + RuntimeStaleSecs float64 `json:"runtime_stale_secs"` + MaxPerTick int32 `json:"max_per_tick"` +} + +// Selects (and row-locks) up to @max_per_tick tasks stuck in dispatched/running +// beyond the given thresholds. Candidate half of the MUL-4332 poison-isolated +// fail path (review point 2): the caller resolves each task's workspace, fails +// only the resolvable set via FailAgentTasksByIDs and emits its task.failed event +// in the SAME transaction. FOR UPDATE SKIP LOCKED keeps the backstop off rows a +// daemon is actively claiming; the LIMIT bounds the lock hold. +// +// Each branch pairs a wall-clock deadline with a task-appropriate liveness +// signal, so the sweeper only kills tasks whose owning daemon is no longer +// proving it is alive: +// +// - Dispatched: `prepare_lease_expires_at` is refreshed every 15s by the +// daemon between claim and StartTask (see startTaskPrepareLeaseExtender). +// A live lease excludes the row. +// +// - Running: no per-task lease is renewed once StartTask fires, so we key +// off the daemon-wide heartbeat instead — `agent_runtime.last_seen_at`, +// which the daemon bumps every ~15s while it is up. A running task whose +// runtime is `online` AND whose `last_seen_at` is within +// @runtime_stale_secs is treated as alive and is NOT killed by this +// wall-clock backstop, even after `started_at` exceeds the running +// timeout. This is what lets healthy multi-hour research / training runs +// survive on self-hosted deployments (MUL-4107): the daemon side is +// bounded only by inactivity watchdogs (idle / per-tool), so the +// server-side wall clock must not shadow that with a coarser cap. +// +// The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` +// (which mixes DB `last_seen_at` with the Redis LivenessStore and fails +// offline-runtime tasks via SelectTasksForOfflineRuntimes in the same tick). +// The wall-clock branch +// here is a defensive backstop for pathological cases where a runtime row +// somehow retains status='online' with a stale DB heartbeat for longer than +// the wall clock allows. +// +// runtime_id IS NULL: a running row with no runtime is by definition not +// proving liveness, so the wall clock is allowed to fire — same shape as +// the legacy pure-wall-clock behavior for that (rare / historical) case. +// +// waiting_local_directory rows are intentionally excluded: the daemon owns +// the wait (with its own ctx-driven timeout) and a legitimate queue ahead +// of this task can exceed the dispatch / running timeouts without being +// "stuck". If the daemon dies, SelectOrphanedTasksForRuntime reclaims +// those rows at restart. +func (q *Queries) SelectStaleTasksToFail(ctx context.Context, arg SelectStaleTasksToFailParams) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectStaleTasksToFail, + arg.DispatchTimeoutSecs, + arg.RunningTimeoutSecs, + arg.RuntimeStaleSecs, + arg.MaxPerTick, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const setTaskDeliveredCommentIDs = `-- name: SetTaskDeliveredCommentIDs :one UPDATE agent_task_queue SET delivered_comment_ids = $1::uuid[] diff --git a/server/pkg/db/generated/issue.sql.go b/server/pkg/db/generated/issue.sql.go index 689af304004..84392e10967 100644 --- a/server/pkg/db/generated/issue.sql.go +++ b/server/pkg/db/generated/issue.sql.go @@ -1159,34 +1159,61 @@ func (q *Queries) LockIssueDuplicateKey(ctx context.Context, dollar_1 string) er return err } -const lockIssueStatusForEvent = `-- name: LockIssueStatusForEvent :one -SELECT status, assignee_type, assignee_id -FROM issue +const lockIssueRowForUpdate = `-- name: LockIssueRowForUpdate :one +SELECT id, workspace_id, title, description, status, priority, assignee_type, assignee_id, creator_type, creator_id, parent_issue_id, acceptance_criteria, context_refs, position, due_date, created_at, updated_at, number, project_id, origin_type, origin_id, first_executed_at, start_date, metadata, stage, properties FROM issue WHERE id = $1 AND workspace_id = $2 FOR UPDATE ` -type LockIssueStatusForEventParams struct { +type LockIssueRowForUpdateParams struct { ID pgtype.UUID `json:"id"` WorkspaceID pgtype.UUID `json:"workspace_id"` } -type LockIssueStatusForEventRow struct { - Status string `json:"status"` - AssigneeType pgtype.Text `json:"assignee_type"` - AssigneeID pgtype.UUID `json:"assignee_id"` -} - -// Row-locks an issue and returns its authoritative status + assignee for -// domain-event emission (MUL-4332 review point 3). Callers read this INSIDE the -// update transaction, immediately before UpdateIssue/UpdateIssueStatus, so the -// event's `from` reflects the truly-current row rather than a snapshot read -// outside the tx — two concurrent transitions then serialize on the lock and -// each records the correct edge instead of both reporting the same stale `from`. -func (q *Queries) LockIssueStatusForEvent(ctx context.Context, arg LockIssueStatusForEventParams) (LockIssueStatusForEventRow, error) { - row := q.db.QueryRow(ctx, lockIssueStatusForEvent, arg.ID, arg.WorkspaceID) - var i LockIssueStatusForEventRow - err := row.Scan(&i.Status, &i.AssigneeType, &i.AssigneeID) +// Row-locks an issue and returns its full authoritative row for both a +// concurrency-safe update and domain-event emission (MUL-4332 review points 3 & +// 1). Callers read this INSIDE the update transaction, immediately before +// UpdateIssue/UpdateIssueStatus. Two purposes: +// 1. The event's `from` (status/assignee) reflects the truly-current row +// rather than a pre-tx snapshot, so concurrent transitions serialize on +// the lock and each records the correct edge instead of a stale `from`. +// 2. UpdateIssue writes the nullable columns (assignee, dates, parent, +// project, stage) as bare narg — an untouched field carries whatever the +// caller pre-filled. Pre-filling from a pre-tx snapshot lets a concurrent +// writer's change be silently rolled back; callers instead rebuild every +// untouched field from THIS locked row so an unrelated update never clobbers +// a field it did not intend to change. +func (q *Queries) LockIssueRowForUpdate(ctx context.Context, arg LockIssueRowForUpdateParams) (Issue, error) { + row := q.db.QueryRow(ctx, lockIssueRowForUpdate, arg.ID, arg.WorkspaceID) + var i Issue + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Title, + &i.Description, + &i.Status, + &i.Priority, + &i.AssigneeType, + &i.AssigneeID, + &i.CreatorType, + &i.CreatorID, + &i.ParentIssueID, + &i.AcceptanceCriteria, + &i.ContextRefs, + &i.Position, + &i.DueDate, + &i.CreatedAt, + &i.UpdatedAt, + &i.Number, + &i.ProjectID, + &i.OriginType, + &i.OriginID, + &i.FirstExecutedAt, + &i.StartDate, + &i.Metadata, + &i.Stage, + &i.Properties, + ) return i, err } diff --git a/server/pkg/db/generated/runtime.sql.go b/server/pkg/db/generated/runtime.sql.go index e2999d62d18..18c2e3734e7 100644 --- a/server/pkg/db/generated/runtime.sql.go +++ b/server/pkg/db/generated/runtime.sql.go @@ -217,89 +217,6 @@ func (q *Queries) DeleteSystemAgentsByRuntime(ctx context.Context, runtimeID pgt return err } -const failTasksForOfflineRuntimes = `-- name: FailTasksForOfflineRuntimes :many -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'runtime went offline', - failure_reason = 'runtime_offline', - wait_reason = NULL -WHERE status IN ('dispatched', 'running', 'waiting_local_directory') - AND runtime_id IN ( - SELECT id FROM agent_runtime WHERE status = 'offline' - ) -RETURNING id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id -` - -// Marks dispatched/running/waiting_local_directory tasks as failed when -// their runtime is offline. This cleans up orphaned tasks after a daemon -// crash or network partition. -func (q *Queries) FailTasksForOfflineRuntimes(ctx context.Context) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, failTasksForOfflineRuntimes) - if err != nil { - return nil, err - } - defer rows.Close() - items := []AgentTaskQueue{} - for rows.Next() { - var i AgentTaskQueue - if err := rows.Scan( - &i.ID, - &i.AgentID, - &i.IssueID, - &i.Status, - &i.Priority, - &i.DispatchedAt, - &i.StartedAt, - &i.CompletedAt, - &i.Result, - &i.Error, - &i.CreatedAt, - &i.Context, - &i.RuntimeID, - &i.SessionID, - &i.WorkDir, - &i.TriggerCommentID, - &i.ChatSessionID, - &i.AutopilotRunID, - &i.Attempt, - &i.MaxAttempts, - &i.ParentTaskID, - &i.FailureReason, - &i.TriggerSummary, - &i.ForceFreshSession, - &i.IsLeaderTask, - &i.WaitReason, - &i.InitiatorUserID, - &i.HandoffNote, - &i.PrepareLeaseExpiresAt, - &i.SquadID, - &i.RuntimeMcpOverlay, - &i.EscalationForTaskID, - &i.FireAt, - &i.OriginatorUserID, - &i.RuntimeConnectedApps, - &i.CoalescedCommentIds, - &i.DeliveredCommentIds, - &i.ChatInputTaskID, - &i.ChatFinalizeDeferredAt, - &i.OriginatorSource, - &i.DelegatedFromTaskID, - &i.RetryOfTaskID, - &i.RerunOfTaskID, - &i.RuleVersionID, - &i.TriggerEvidenceKind, - &i.TriggerEvidenceRefID, - &i.AccountableUserID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const findLegacyRuntimesByDaemonID = `-- name: FindLegacyRuntimesByDaemonID :many SELECT id, workspace_id, daemon_id, name, runtime_mode, provider, status, device_info, metadata, last_seen_at, created_at, updated_at, owner_id, legacy_daemon_id, visibility, profile_id, custom_name FROM agent_runtime WHERE workspace_id = $1 @@ -950,6 +867,94 @@ func (q *Queries) SelectStaleOnlineRuntimes(ctx context.Context, staleSeconds fl return items, nil } +const selectTasksForOfflineRuntimes = `-- name: SelectTasksForOfflineRuntimes :many +SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue +WHERE status IN ('dispatched', 'running', 'waiting_local_directory') + AND runtime_id IN ( + SELECT id FROM agent_runtime WHERE status = 'offline' + ) +ORDER BY created_at ASC +LIMIT $1::int +FOR UPDATE SKIP LOCKED +` + +// Selects (and row-locks) up to @max_per_tick dispatched/running/ +// waiting_local_directory tasks whose runtime is offline — the orphans a daemon +// crash or network partition leaves behind. This is the candidate half of the +// MUL-4332 poison-isolated fail path (review point 2): the caller resolves each +// task's workspace, fails only the resolvable set via FailAgentTasksByIDs and +// emits its task.failed event in the SAME transaction, so an unresolvable poison +// row is skipped instead of rolling back the whole batch. FOR UPDATE SKIP LOCKED +// keeps the sweeper off rows a daemon is actively claiming; the LIMIT bounds the +// lock hold and the rest drain on later ticks. +func (q *Queries) SelectTasksForOfflineRuntimes(ctx context.Context, maxPerTick int32) ([]AgentTaskQueue, error) { + rows, err := q.db.Query(ctx, selectTasksForOfflineRuntimes, maxPerTick) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AgentTaskQueue{} + for rows.Next() { + var i AgentTaskQueue + if err := rows.Scan( + &i.ID, + &i.AgentID, + &i.IssueID, + &i.Status, + &i.Priority, + &i.DispatchedAt, + &i.StartedAt, + &i.CompletedAt, + &i.Result, + &i.Error, + &i.CreatedAt, + &i.Context, + &i.RuntimeID, + &i.SessionID, + &i.WorkDir, + &i.TriggerCommentID, + &i.ChatSessionID, + &i.AutopilotRunID, + &i.Attempt, + &i.MaxAttempts, + &i.ParentTaskID, + &i.FailureReason, + &i.TriggerSummary, + &i.ForceFreshSession, + &i.IsLeaderTask, + &i.WaitReason, + &i.InitiatorUserID, + &i.HandoffNote, + &i.PrepareLeaseExpiresAt, + &i.SquadID, + &i.RuntimeMcpOverlay, + &i.EscalationForTaskID, + &i.FireAt, + &i.OriginatorUserID, + &i.RuntimeConnectedApps, + &i.CoalescedCommentIds, + &i.DeliveredCommentIds, + &i.ChatInputTaskID, + &i.ChatFinalizeDeferredAt, + &i.OriginatorSource, + &i.DelegatedFromTaskID, + &i.RetryOfTaskID, + &i.RerunOfTaskID, + &i.RuleVersionID, + &i.TriggerEvidenceKind, + &i.TriggerEvidenceRefID, + &i.AccountableUserID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const setAgentRuntimeOffline = `-- name: SetAgentRuntimeOffline :exec UPDATE agent_runtime SET status = 'offline', updated_at = now() diff --git a/server/pkg/db/queries/agent.sql b/server/pkg/db/queries/agent.sql index e817ebf8b70..53c27a0dd3a 100644 --- a/server/pkg/db/queries/agent.sql +++ b/server/pkg/db/queries/agent.sql @@ -698,25 +698,31 @@ SET session_id = COALESCE(sqlc.narg('session_id'), session_id), work_dir = COALESCE(sqlc.narg('work_dir'), work_dir) WHERE id = $1 AND status IN ('dispatched', 'running'); --- name: RecoverOrphanedTasksForRuntime :many --- Called by the daemon at startup. Atomically fails any dispatched/running/ --- waiting_local_directory task that the prior incarnation of this runtime --- owned but did not finalize. Returns the failed rows so callers can hand --- them to the auto-retry path. waiting_local_directory rows are included --- because the daemon holding the path lock is the same process that just --- died — without us, the row would sit waiting forever. -UPDATE agent_task_queue -SET status = 'failed', - completed_at = now(), - error = 'daemon restarted while task was in flight', - failure_reason = 'runtime_recovery', - wait_reason = NULL, - prepare_lease_expires_at = NULL -WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') -RETURNING *; - --- name: FailStaleTasks :many --- Fails tasks stuck in dispatched/running beyond the given thresholds. +-- name: SelectOrphanedTasksForRuntime :many +-- Called by the daemon at startup. Selects (and row-locks) up to @max_per_tick +-- dispatched/running/waiting_local_directory tasks that the prior incarnation of +-- this runtime owned but did not finalize. waiting_local_directory rows are +-- included because the daemon holding the path lock is the same process that +-- just died — without us, the row would sit waiting forever. +-- +-- Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +-- caller resolves each task's workspace, fails only the resolvable set via +-- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +-- FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's +-- own claim path; the LIMIT bounds the lock hold. +SELECT * FROM agent_task_queue +WHERE runtime_id = @runtime_id AND status IN ('dispatched', 'running', 'waiting_local_directory') +ORDER BY created_at ASC +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; + +-- name: SelectStaleTasksToFail :many +-- Selects (and row-locks) up to @max_per_tick tasks stuck in dispatched/running +-- beyond the given thresholds. Candidate half of the MUL-4332 poison-isolated +-- fail path (review point 2): the caller resolves each task's workspace, fails +-- only the resolvable set via FailAgentTasksByIDs and emits its task.failed event +-- in the SAME transaction. FOR UPDATE SKIP LOCKED keeps the backstop off rows a +-- daemon is actively claiming; the LIMIT bounds the lock hold. -- -- Each branch pairs a wall-clock deadline with a task-appropriate liveness -- signal, so the sweeper only kills tasks whose owning daemon is no longer @@ -738,8 +744,9 @@ RETURNING *; -- server-side wall clock must not shadow that with a coarser cap. -- -- The daemon-dead case is the primary responsibility of `sweepStaleRuntimes` --- (which mixes DB `last_seen_at` with the Redis LivenessStore and calls --- `FailTasksForOfflineRuntimes` in the same tick). The wall-clock branch +-- (which mixes DB `last_seen_at` with the Redis LivenessStore and fails +-- offline-runtime tasks via SelectTasksForOfflineRuntimes in the same tick). +-- The wall-clock branch -- here is a defensive backstop for pathological cases where a runtime row -- somehow retains status='online' with a stale DB heartbeat for longer than -- the wall clock allows. @@ -751,12 +758,9 @@ RETURNING *; -- waiting_local_directory rows are intentionally excluded: the daemon owns -- the wait (with its own ctx-driven timeout) and a legitimate queue ahead -- of this task can exceed the dispatch / running timeouts without being --- "stuck". If the daemon dies, RecoverOrphanedTasksForRuntime reclaims +-- "stuck". If the daemon dies, SelectOrphanedTasksForRuntime reclaims -- those rows at restart. -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'task timed out', - failure_reason = 'timeout', - prepare_lease_expires_at = NULL +SELECT * FROM agent_task_queue WHERE ( status = 'dispatched' AND dispatched_at < now() - make_interval(secs => @dispatch_timeout_secs::double precision) @@ -772,50 +776,50 @@ WHERE ( AND r.last_seen_at >= now() - make_interval(secs => @runtime_stale_secs::double precision) ) ) -RETURNING *; - --- name: ExpireStaleQueuedTasks :many --- Fails tasks that have been sitting in 'queued' for longer than the TTL. --- This is the cleanup arm of the MUL-1899 "queued backlog" fix: even with the --- new dispatch-time admission gate that refuses to enqueue when the runtime --- is offline, we still need to drain the historical 87k+ doomed rows and --- handle edge cases where a runtime goes offline AFTER a task is already --- queued (the admission check protects new enqueues, not in-flight queue --- depth). +ORDER BY started_at ASC NULLS FIRST, dispatched_at ASC NULLS FIRST +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; + +-- name: SelectExpiredQueuedTasks :many +-- Selects (and row-locks) up to @max_per_tick tasks sitting in 'queued' past the +-- TTL — the cleanup arm of the MUL-1899 "queued backlog" fix (drains the +-- historical 87k+ doomed rows and catches the case where a runtime goes offline +-- AFTER a task is already queued). -- --- Concurrency safety: the daemon's claim path may race with this sweeper to --- transition the same row out of 'queued'. We protect against that two --- ways: --- 1. The CTE selects victims with FOR UPDATE SKIP LOCKED so a row that is --- currently being claimed (or otherwise locked) is skipped — no lock --- contention with the dispatch path, and we won't queue up behind it. --- 2. The outer UPDATE re-checks status='queued' AND the TTL predicate at --- apply time. If a daemon claimed the row between selection and update --- (e.g. lock released after the claim transaction commits), the row is --- already 'dispatched'/'running' and the WHERE clause filters it out --- so we cannot clobber an in-flight task. --- Capped via LIMIT inside the CTE so a single sweep tick cannot monopolise --- the DB when the backlog is large — the sweeper drains the rest on --- subsequent ticks. -WITH victims AS ( - SELECT id FROM agent_task_queue - WHERE status = 'queued' - AND created_at < now() - make_interval(secs => @ttl_secs::double precision) - ORDER BY created_at ASC - LIMIT @max_per_tick::int - FOR UPDATE SKIP LOCKED -) -UPDATE agent_task_queue t +-- Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the +-- caller resolves each task's workspace, fails only the resolvable set via +-- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. +-- FOR UPDATE SKIP LOCKED skips rows a daemon is currently claiming, so we never +-- contend with the dispatch path or clobber an in-flight task; because the fail +-- runs in this same transaction the locked rows cannot transition out of 'queued' +-- underneath us. The LIMIT bounds the lock hold; the rest drain on later ticks. +SELECT * FROM agent_task_queue +WHERE status = 'queued' + AND created_at < now() - make_interval(secs => @ttl_secs::double precision) +ORDER BY created_at ASC +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; + +-- name: FailAgentTasksByIDs :many +-- Fails a specific, already-resolved set of tasks by id — the terminal half of +-- the MUL-4332 poison-isolated bulk fail (review point 2). The caller has just +-- selected these ids with SelectStaleTasksToFail / SelectExpiredQueuedTasks / +-- SelectTasksForOfflineRuntimes / SelectOrphanedTasksForRuntime FOR UPDATE in +-- the SAME transaction, so the rows are locked and cannot have transitioned; +-- the status guard is a defensive backstop that keeps this idempotent if the id +-- set is ever reused. @error / @failure_reason are supplied per sweeper. Both +-- wait_reason and prepare_lease_expires_at are cleared (clearing a NULL is a +-- no-op) so this one query serves every bulk-fail caller. +UPDATE agent_task_queue SET status = 'failed', completed_at = now(), - error = 'task expired in queue', - failure_reason = 'queued_expired', + error = @error, + failure_reason = @failure_reason, + wait_reason = NULL, prepare_lease_expires_at = NULL -FROM victims v -WHERE t.id = v.id - AND t.status = 'queued' - AND t.created_at < now() - make_interval(secs => @ttl_secs::double precision) -RETURNING t.*; +WHERE id = ANY(@ids::uuid[]) + AND status IN ('queued', 'dispatched', 'running', 'waiting_local_directory') +RETURNING *; -- name: CancelAgentTask :one UPDATE agent_task_queue diff --git a/server/pkg/db/queries/issue.sql b/server/pkg/db/queries/issue.sql index fd0f04cf85c..f9437f8ee6e 100644 --- a/server/pkg/db/queries/issue.sql +++ b/server/pkg/db/queries/issue.sql @@ -362,14 +362,20 @@ SET first_executed_at = now() WHERE id = $1 AND first_executed_at IS NULL RETURNING id, workspace_id, creator_type, creator_id, first_executed_at; --- name: LockIssueStatusForEvent :one --- Row-locks an issue and returns its authoritative status + assignee for --- domain-event emission (MUL-4332 review point 3). Callers read this INSIDE the --- update transaction, immediately before UpdateIssue/UpdateIssueStatus, so the --- event's `from` reflects the truly-current row rather than a snapshot read --- outside the tx — two concurrent transitions then serialize on the lock and --- each records the correct edge instead of both reporting the same stale `from`. -SELECT status, assignee_type, assignee_id -FROM issue +-- name: LockIssueRowForUpdate :one +-- Row-locks an issue and returns its full authoritative row for both a +-- concurrency-safe update and domain-event emission (MUL-4332 review points 3 & +-- 1). Callers read this INSIDE the update transaction, immediately before +-- UpdateIssue/UpdateIssueStatus. Two purposes: +-- 1. The event's `from` (status/assignee) reflects the truly-current row +-- rather than a pre-tx snapshot, so concurrent transitions serialize on +-- the lock and each records the correct edge instead of a stale `from`. +-- 2. UpdateIssue writes the nullable columns (assignee, dates, parent, +-- project, stage) as bare narg — an untouched field carries whatever the +-- caller pre-filled. Pre-filling from a pre-tx snapshot lets a concurrent +-- writer's change be silently rolled back; callers instead rebuild every +-- untouched field from THIS locked row so an unrelated update never clobbers +-- a field it did not intend to change. +SELECT * FROM issue WHERE id = $1 AND workspace_id = $2 FOR UPDATE; diff --git a/server/pkg/db/queries/runtime.sql b/server/pkg/db/queries/runtime.sql index 6daf95f347d..9f397b6a6f3 100644 --- a/server/pkg/db/queries/runtime.sql +++ b/server/pkg/db/queries/runtime.sql @@ -226,19 +226,24 @@ WHERE status = 'online' AND last_seen_at < now() - make_interval(secs => @stale_seconds::double precision) RETURNING id, workspace_id, owner_id, daemon_id, provider; --- name: FailTasksForOfflineRuntimes :many --- Marks dispatched/running/waiting_local_directory tasks as failed when --- their runtime is offline. This cleans up orphaned tasks after a daemon --- crash or network partition. -UPDATE agent_task_queue -SET status = 'failed', completed_at = now(), error = 'runtime went offline', - failure_reason = 'runtime_offline', - wait_reason = NULL +-- name: SelectTasksForOfflineRuntimes :many +-- Selects (and row-locks) up to @max_per_tick dispatched/running/ +-- waiting_local_directory tasks whose runtime is offline — the orphans a daemon +-- crash or network partition leaves behind. This is the candidate half of the +-- MUL-4332 poison-isolated fail path (review point 2): the caller resolves each +-- task's workspace, fails only the resolvable set via FailAgentTasksByIDs and +-- emits its task.failed event in the SAME transaction, so an unresolvable poison +-- row is skipped instead of rolling back the whole batch. FOR UPDATE SKIP LOCKED +-- keeps the sweeper off rows a daemon is actively claiming; the LIMIT bounds the +-- lock hold and the rest drain on later ticks. +SELECT * FROM agent_task_queue WHERE status IN ('dispatched', 'running', 'waiting_local_directory') AND runtime_id IN ( SELECT id FROM agent_runtime WHERE status = 'offline' ) -RETURNING *; +ORDER BY created_at ASC +LIMIT @max_per_tick::int +FOR UPDATE SKIP LOCKED; -- name: ListAgentRuntimesByOwner :many SELECT * FROM agent_runtime From e657288fbd9f1a9b2c64bf0fe718864a2620a65a Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 13:33:11 +0800 Subject: [PATCH 04/15] fix(automation): drain recover-orphans + retry_eligible contract (MUL-4332 PR1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon 3rd-round review (2 must-fixes): 1. recover-orphans now drains every orphan for a runtime instead of capping at one 500-row page. Registration flips the runtime back online, so the offline sweep never reaps rows left past the first page; SelectOrphanedTasksForRuntime gains a (created_at, id) keyset cursor, the handler returns has_more + next_cursor per page, and the daemon client loops until drained. The cursor advances over skipped poison rows so a page of poison at the front can't stall the drain. Tests: handler limit+1 end-to-end drain, service poison-next-page. 2. task.failed.retryable renamed to retry_eligible with atomic, decidable semantics: it reports only that the failure met the eligibility predicate at commit time, computed in the same tx as the fail+event on every path. It no longer promises a fresh attempt will arrive — the bulk paths create the retry child best-effort after commit. Dropped the "not yet terminal, a fresh attempt is coming" consumer guarantee. Test: retry child crash-window contract. Co-authored-by: multica-agent --- server/internal/daemon/client.go | 48 +++++- server/internal/domainevent/event.go | 30 ++-- server/internal/domainevent/event_test.go | 2 +- .../handler/recover_orphans_drain_test.go | 121 +++++++++++++++ server/internal/handler/task_lifecycle.go | 140 ++++++++++++++---- .../service/domain_event_task_failed_test.go | 110 +++++++++++++- .../service/recover_orphans_cursor_test.go | 134 +++++++++++++++++ server/internal/service/task.go | 33 +++-- server/pkg/db/generated/agent.sql.go | 33 ++++- server/pkg/db/queries/agent.sql | 18 ++- 10 files changed, 599 insertions(+), 70 deletions(-) create mode 100644 server/internal/handler/recover_orphans_drain_test.go create mode 100644 server/internal/service/recover_orphans_cursor_test.go diff --git a/server/internal/daemon/client.go b/server/internal/daemon/client.go index 196866f65c9..b1f960f9716 100644 --- a/server/internal/daemon/client.go +++ b/server/internal/daemon/client.go @@ -421,11 +421,51 @@ func (c *Client) PinTaskSession(ctx context.Context, taskID, sessionID, workDir return c.postJSON(ctx, fmt.Sprintf("/api/daemon/tasks/%s/session", taskID), body, nil) } -// RecoverOrphans tells the server to fail any dispatched/running tasks the -// previous daemon process for this runtime left behind. The server will -// auto-retry eligible tasks. +// maxRecoverOrphanPages bounds the drain loop below. Each page fails up to +// orphanRecoveryBatchSize (500) rows, so this covers ~500k orphaned rows per +// registration — far beyond any real restart. It is purely a backstop so a buggy +// server response (has_more stuck true) can never spin forever; any residual is +// reclaimed by the next registration. +const maxRecoverOrphanPages = 1000 + +// RecoverOrphans tells the server to fail any dispatched/running tasks the previous +// daemon process for this runtime left behind, and to auto-retry the eligible ones. +// +// It DRAINS the runtime's orphans across pages (MUL-4332 review round 3, point 1): +// registration has already upserted the runtime back to `online`, so the server's +// every-tick offline sweep will not reap anything a single capped call leaves +// behind. We therefore loop, threading the server's keyset cursor, until the server +// reports no more pages. The cursor advances over poison (unresolvable) rows the +// server skips, so a page of poison at the front cannot stall the drain. func (c *Client) RecoverOrphans(ctx context.Context, runtimeID string) error { - return c.postJSON(ctx, fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID), map[string]any{}, nil) + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + body := map[string]any{} + for page := 0; page < maxRecoverOrphanPages; page++ { + var resp struct { + HasMore bool `json:"has_more"` + NextCursorCreatedAt string `json:"next_cursor_created_at"` + NextCursorID string `json:"next_cursor_id"` + } + if err := c.postJSON(ctx, path, body, &resp); err != nil { + // An empty 200 body (an older server that predates paging, or a proxy) + // decodes to io.EOF. Treat it as a single non-paged recovery: there is + // no cursor to continue with, so stop cleanly rather than erroring. + if errors.Is(err, io.EOF) { + return nil + } + return err + } + // Stop on the last page, or defensively if the server signalled more but + // gave us no cursor to advance (never expected — avoids re-draining page 0). + if !resp.HasMore || resp.NextCursorCreatedAt == "" || resp.NextCursorID == "" { + return nil + } + body = map[string]any{ + "cursor_created_at": resp.NextCursorCreatedAt, + "cursor_id": resp.NextCursorID, + } + } + return nil } // GetTaskStatus returns the current status of a task. Used by the daemon to diff --git a/server/internal/domainevent/event.go b/server/internal/domainevent/event.go index 0bf8e86e0e8..21a48ad844c 100644 --- a/server/internal/domainevent/event.go +++ b/server/internal/domainevent/event.go @@ -166,19 +166,25 @@ func (TaskCompletedPayload) eventType() string { return TypeTaskCompleted } func (TaskCompletedPayload) subjectType() string { return SubjectTask } func (TaskCompletedPayload) schemaVersion() int32 { return 1 } -// TaskFailedPayload — subject is the task. Retryable reports whether this failure -// is eligible for an automatic retry (an infra-shaped reason still within the -// attempt budget). The single FailTask path creates the retry child in the SAME -// transaction, so there it is exact; the bulk sweeper paths create it immediately -// after commit, so there it means "a retry is expected" — both derive it from the -// shared retryEligible predicate so the event never contradicts the actual retry -// decision. A consumer should read retryable=true as "not yet terminal, a fresh -// attempt is coming" rather than reacting to the failure. +// TaskFailedPayload — subject is the task. RetryEligible is an atomically-decidable +// fact: at the instant this failure committed, the task satisfied the auto-retry +// eligibility predicate (an infra-shaped reason, still within the attempt budget, +// not an autopilot run, and reporting to an issue or chat). Every path derives it +// from the shared retryEligible predicate in the SAME transaction as the fail + its +// event, so the flag can never contradict the eligibility decision. +// +// It deliberately does NOT promise that a retry child was — or ever will be — +// created. The single FailTask path does create the child in this same transaction, +// but the bulk sweeper / orphan-recovery paths create it best-effort AFTER commit, +// where CreateRetryTask can return an error or the process can crash between the two +// steps. So a consumer must treat task.failed as terminal for the subject task and +// read retry_eligible only as "this failure was retry-eligible", never as "a fresh +// attempt is guaranteed to arrive". type TaskFailedPayload struct { - IssueID string `json:"issue_id,omitempty"` - AgentID string `json:"agent_id,omitempty"` - Retryable bool `json:"retryable"` - ErrorCode string `json:"error_code,omitempty"` + IssueID string `json:"issue_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + RetryEligible bool `json:"retry_eligible"` + ErrorCode string `json:"error_code,omitempty"` } func (TaskFailedPayload) eventType() string { return TypeTaskFailed } diff --git a/server/internal/domainevent/event_test.go b/server/internal/domainevent/event_test.go index 10fbb73ff8c..3e6e9d80481 100644 --- a/server/internal/domainevent/event_test.go +++ b/server/internal/domainevent/event_test.go @@ -33,7 +33,7 @@ func TestConstructorsProduceValidEnvelopes(t *testing.T) { {"issue.assigned", IssueAssigned(ws, subj, actor, IssueAssignedPayload{ToAssigneeType: "agent"}), TypeIssueAssigned, SubjectIssue}, {"comment.created", CommentCreated(ws, subj, actor, CommentCreatedPayload{IssueID: uuid.NewString(), AuthorType: "member"}), TypeCommentCreated, SubjectComment}, {"task.completed", TaskCompleted(ws, subj, SystemActor(), TaskCompletedPayload{IssueID: uuid.NewString()}), TypeTaskCompleted, SubjectTask}, - {"task.failed", TaskFailed(ws, subj, SystemActor(), TaskFailedPayload{Retryable: true}), TypeTaskFailed, SubjectTask}, + {"task.failed", TaskFailed(ws, subj, SystemActor(), TaskFailedPayload{RetryEligible: true}), TypeTaskFailed, SubjectTask}, } for _, tc := range cases { diff --git a/server/internal/handler/recover_orphans_drain_test.go b/server/internal/handler/recover_orphans_drain_test.go new file mode 100644 index 00000000000..a7625a7daaa --- /dev/null +++ b/server/internal/handler/recover_orphans_drain_test.go @@ -0,0 +1,121 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +// recover-orphans must DRAIN every orphaned task for a runtime, not just the first +// page (MUL-4332 review round 3, point 1). Registration flips the runtime back +// online, so the offline sweep will not reap anything a single capped call leaves +// behind — the daemon therefore pages until has_more is false. We seed +// orphanRecoveryBatchSize + 1 orphans (one past a single page) and drive the real +// handler exactly as the daemon client does: thread the keyset cursor from each +// response into the next request. All of them must end failed, each with one +// task.failed event, across exactly two pages. +func TestRecoverOrphansDrainsPastBatchLimit(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + + // A dedicated runtime + agent in the handler-test workspace so the daemon token + // (scoped to testWorkspaceID) authorizes recover-orphans for it, isolated from + // the shared fixture runtime. + var runtimeID, agentID string + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent_runtime (workspace_id, name, runtime_mode, provider, status, device_info, metadata, owner_id) + VALUES ($1, 'orphan-drain-runtime', 'cloud', 'codex', 'online', '', '{}'::jsonb, $2) + RETURNING id`, testWorkspaceID, testUserID).Scan(&runtimeID); err != nil { + t.Fatalf("seed runtime: %v", err) + } + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent (workspace_id, name, runtime_mode, runtime_config, runtime_id, visibility, + max_concurrent_tasks, owner_id, instructions, custom_env, custom_args) + VALUES ($1, 'orphan-drain-agent', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3, '', '{}'::jsonb, '[]'::jsonb) + RETURNING id`, testWorkspaceID, runtimeID, testUserID).Scan(&agentID); err != nil { + t.Fatalf("seed agent: %v", err) + } + t.Cleanup(func() { + // agent_runtime → agent → agent_task_queue cascade on delete cleans the rows; + // task.failed events carry no FK, so drop them explicitly first. + testPool.Exec(context.Background(), + `DELETE FROM domain_event WHERE subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, runtimeID) + testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID) + }) + + // Seed one past a single page. attempt=1/max=2 with no issue/chat means the row + // is NOT retry-eligible (retryEligible needs an issue or chat), so the shared + // post-fail pipeline neither auto-retries nor resets an issue — the assertion is + // about pagination alone. A single INSERT gives them all the same created_at, so + // this also exercises the keyset id tiebreaker. + total := orphanRecoveryBatchSize + 1 + if _, err := testPool.Exec(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, status, priority) + SELECT $1, $2, 'running', 0 FROM generate_series(1, $3)`, agentID, runtimeID, total); err != nil { + t.Fatalf("seed orphaned tasks: %v", err) + } + + // Drive the real handler in a drain loop, threading the cursor like the daemon. + const daemonID = "recover-drain-test" + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + var body any + totalFailed, pages := 0, 0 + for { + pages++ + if pages > 10 { + t.Fatalf("drain did not terminate after %d pages", pages) + } + w := httptest.NewRecorder() + req := withURLParam(newDaemonTokenRequest(http.MethodPost, path, body, testWorkspaceID, daemonID), "runtimeId", runtimeID) + testHandler.RecoverOrphanedTasks(w, req) + if w.Code != http.StatusOK { + t.Fatalf("page %d: status %d: %s", pages, w.Code, w.Body.String()) + } + var resp RecoverOrphansResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("page %d decode: %v", pages, err) + } + totalFailed += resp.Orphaned + if !resp.HasMore { + break + } + if resp.NextCursorCreatedAt == "" || resp.NextCursorID == "" { + t.Fatalf("page %d: has_more=true but empty cursor", pages) + } + body = map[string]any{"cursor_created_at": resp.NextCursorCreatedAt, "cursor_id": resp.NextCursorID} + } + + if pages != 2 { + t.Errorf("pages = %d, want 2 (a full page of %d, then the final 1)", pages, orphanRecoveryBatchSize) + } + if totalFailed != total { + t.Errorf("failed across drain = %d, want %d", totalFailed, total) + } + + // No orphan is left selectable, and each failed task has exactly one event. + var remaining int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue + WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory')`, + runtimeID).Scan(&remaining); err != nil { + t.Fatalf("count remaining: %v", err) + } + if remaining != 0 { + t.Errorf("remaining orphans = %d, want 0 (drain must reach the row past the first page)", remaining) + } + var events int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM domain_event + WHERE type = 'task.failed' AND subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, + runtimeID).Scan(&events); err != nil { + t.Fatalf("count events: %v", err) + } + if events != total { + t.Errorf("task.failed events = %d, want %d (fact ⇔ event, one per failed row)", events, total) + } +} diff --git a/server/internal/handler/task_lifecycle.go b/server/internal/handler/task_lifecycle.go index f4648fcaa09..0bd835a6b9e 100644 --- a/server/internal/handler/task_lifecycle.go +++ b/server/internal/handler/task_lifecycle.go @@ -6,6 +6,7 @@ import ( "io" "log/slog" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" @@ -14,37 +15,85 @@ import ( ) // orphanRecoveryBatchSize bounds how many orphaned tasks one recover-orphans -// call fails (MUL-4332 review point 2). A single daemon's in-flight concurrency -// is small, so this is far above any realistic per-restart orphan count; the rare -// overflow is caught by the runtime sweeper's every-tick offline pass. +// PAGE fails (MUL-4332 review point 2). Registration upserts the runtime back to +// `online`, so the every-tick offline sweep will NOT reap anything this call leaves +// behind (review round 3, point 1); the daemon therefore drains all pages via the +// keyset cursor rather than relying on a follow-up sweep. This bound only caps the +// lock hold and event volume per page. const orphanRecoveryBatchSize = 500 -// RecoverOrphanedTasks is called by the daemon at startup for each runtime -// it owns. It atomically fails any dispatched/running tasks the server still -// believes belong to that runtime — those are the tasks the previous daemon -// process was running when it died — and triggers MaybeRetryFailedTask for -// each so the user sees a fresh attempt instead of a permanently stuck row. +// RecoverOrphansRequest is the optional body of POST recover-orphans. Empty on the +// first page; the daemon threads back the keyset cursor from the prior page's +// response to fetch the next one. Both fields move together — a partial cursor is +// rejected. +type RecoverOrphansRequest struct { + CursorCreatedAt string `json:"cursor_created_at,omitempty"` + CursorID string `json:"cursor_id,omitempty"` +} + +// RecoverOrphansResponse reports one page of recovery. HasMore + NextCursor* let the +// daemon drain the runtime's orphans across pages; NextCursor* advances over every +// candidate this page locked (failed OR skipped-poison), so a page of poison at the +// front cannot pin the drain — the next page steps past it (review round 3, point 1). +type RecoverOrphansResponse struct { + Orphaned int `json:"orphaned"` + Retried int `json:"retried"` + Skipped int `json:"skipped"` + HasMore bool `json:"has_more"` + NextCursorCreatedAt string `json:"next_cursor_created_at,omitempty"` + NextCursorID string `json:"next_cursor_id,omitempty"` +} + +// RecoverOrphanedTasks is called by the daemon at startup for each runtime it owns. +// It atomically fails a bounded page of the dispatched/running tasks the server +// still believes belong to that runtime — those the previous daemon process was +// running when it died — emitting each row's task.failed event in the same +// transaction, then runs the shared post-failure pipeline (auto-retry, issue +// rollback, reconcile). The daemon calls it repeatedly, threading the keyset cursor, +// until HasMore is false. // -// This is the targeted fix for "issue stuck at in_progress when daemon -// restarts mid-task": the runtime heartbeat sweeper takes up to 75s + the -// in-process task timeout (2.5h) to notice such tasks; the daemon itself -// knows the moment it comes back up, so we let it report orphan recovery. +// This is the targeted fix for "issue stuck at in_progress when daemon restarts +// mid-task": the runtime heartbeat sweeper takes up to 75s + the in-process task +// timeout (2.5h) to notice such tasks; the daemon knows the moment it comes back up. +// Because registration flips the runtime back online, the offline sweep will not +// catch a row beyond this page (review round 3, point 1) — hence the cursor-driven +// drain instead of a single capped call. func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { runtimeID := chi.URLParam(r, "runtimeId") if _, ok := h.requireDaemonRuntimeAccess(w, r, runtimeID); !ok { return } - // Select the runtime's orphaned tasks and fail the resolvable ones with their - // task.failed events atomically (MUL-4332 review point 2), so daemon-driven - // recovery converges onto the outbox like the runtime sweepers and an - // unresolvable poison row cannot block the rest. + // Optional keyset cursor: absent on the first page, threaded back by the daemon + // for each subsequent page. + var req RecoverOrphansRequest + if r.ContentLength != 0 { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil && err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + afterCreatedAt, afterID, ok := parseOrphanCursor(w, req) + if !ok { + return + } + + // Select one page of the runtime's orphaned tasks and fail the resolvable ones + // with their task.failed events atomically (MUL-4332 review point 2), so + // daemon-driven recovery converges onto the outbox like the runtime sweepers and + // an unresolvable poison row cannot block the rest. We capture the raw candidate + // page (before poison isolation drops rows) to compute has_more and the cursor. + var candidates []db.AgentTaskQueue rows, err := h.TaskService.FailBulkTasksWithEvents(r.Context(), func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - return qtx.SelectOrphanedTasksForRuntime(r.Context(), db.SelectOrphanedTasksForRuntimeParams{ - RuntimeID: parseUUID(runtimeID), - MaxPerTick: orphanRecoveryBatchSize, + c, e := qtx.SelectOrphanedTasksForRuntime(r.Context(), db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: parseUUID(runtimeID), + AfterCreatedAt: afterCreatedAt, + AfterID: afterID, + MaxPerTick: orphanRecoveryBatchSize, }) + candidates = c + return c, e }, func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { return qtx.FailAgentTasksByIDs(r.Context(), db.FailAgentTasksByIDsParams{ @@ -66,18 +115,57 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { // was created (max_attempts exhausted, autopilot, non-retryable reason). retried := h.TaskService.HandleFailedTasks(r.Context(), rows) - if len(rows) > 0 { - slog.Info("recover-orphans completed", + // A full candidate page means there may be more behind it. The cursor advances + // over the LAST candidate we locked (failed or skipped), so the next page never + // re-selects this page — including any poison rows left un-failed at the front. + resp := RecoverOrphansResponse{ + Orphaned: len(rows), + Retried: retried, + Skipped: len(candidates) - len(rows), + HasMore: len(candidates) == orphanRecoveryBatchSize, + } + if resp.HasMore && len(candidates) > 0 { + last := candidates[len(candidates)-1] + resp.NextCursorCreatedAt = last.CreatedAt.Time.UTC().Format(time.RFC3339Nano) + resp.NextCursorID = uuidToString(last.ID) + } + + if len(candidates) > 0 { + slog.Info("recover-orphans page", "runtime_id", runtimeID, - "orphaned", len(rows), + "candidates", len(candidates), + "orphaned", resp.Orphaned, + "skipped", resp.Skipped, "retried", retried, + "has_more", resp.HasMore, ) } - writeJSON(w, http.StatusOK, map[string]any{ - "orphaned": len(rows), - "retried": retried, - }) + writeJSON(w, http.StatusOK, resp) +} + +// parseOrphanCursor turns the optional (cursor_created_at, cursor_id) request pair +// into keyset params. Neither present → the first page (both NULL). Both present → +// the keyset. Exactly one present is a malformed cursor and is rejected, so a bad +// client can never silently restart the drain from the beginning (an infinite loop). +func parseOrphanCursor(w http.ResponseWriter, req RecoverOrphansRequest) (pgtype.Timestamptz, pgtype.UUID, bool) { + if req.CursorCreatedAt == "" && req.CursorID == "" { + return pgtype.Timestamptz{}, pgtype.UUID{}, true + } + if req.CursorCreatedAt == "" || req.CursorID == "" { + writeError(w, http.StatusBadRequest, "cursor_created_at and cursor_id must be set together") + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + ts, err := time.Parse(time.RFC3339Nano, req.CursorCreatedAt) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid cursor_created_at") + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + id, ok := parseUUIDOrBadRequest(w, req.CursorID, "cursor_id") + if !ok { + return pgtype.Timestamptz{}, pgtype.UUID{}, false + } + return pgtype.Timestamptz{Time: ts, Valid: true}, id, true } // PinTaskSession lets the daemon persist the agent's session_id and diff --git a/server/internal/service/domain_event_task_failed_test.go b/server/internal/service/domain_event_task_failed_test.go index 1e9ebe7e3dc..987b3069aa1 100644 --- a/server/internal/service/domain_event_task_failed_test.go +++ b/server/internal/service/domain_event_task_failed_test.go @@ -62,14 +62,14 @@ func TestFailBulkTasksWithEventsEmitsTaskFailed(t *testing.T) { } // Exactly one task.failed event for the task, carrying the issue + error, - // attributed to the platform, with a retryable flag matching the shared + // attributed to the platform, with a retry_eligible flag matching the shared // retryEligible predicate. var evtType, actorType, payload string - var retryable bool + var retryEligibleFlag bool if err := pool.QueryRow(ctx, - `SELECT type, actor_type, (payload->>'retryable')::bool, payload::text + `SELECT type, actor_type, (payload->>'retry_eligible')::bool, payload::text FROM domain_event WHERE subject_type = 'task' AND subject_id = $1`, - taskID).Scan(&evtType, &actorType, &retryable, &payload); err != nil { + taskID).Scan(&evtType, &actorType, &retryEligibleFlag, &payload); err != nil { t.Fatalf("expected a task.failed domain event: %v", err) } if evtType != "task.failed" { @@ -78,8 +78,8 @@ func TestFailBulkTasksWithEventsEmitsTaskFailed(t *testing.T) { if actorType != "system" { t.Errorf("actor_type = %q, want system (a sweeper fail is platform-driven, not the agent's action)", actorType) } - if want := retryEligible("runtime_offline", failed[0]); retryable != want { - t.Errorf("retryable = %v, want %v (event must agree with the auto-retry decision)", retryable, want) + if want := retryEligible("runtime_offline", failed[0]); retryEligibleFlag != want { + t.Errorf("retry_eligible = %v, want %v (event must agree with the eligibility predicate)", retryEligibleFlag, want) } if !strings.Contains(payload, issueID) { t.Errorf("payload %s should carry issue_id %s", payload, issueID) @@ -88,3 +88,101 @@ func TestFailBulkTasksWithEventsEmitsTaskFailed(t *testing.T) { t.Errorf("payload %s should carry the failure reason", payload) } } + +// task.failed.retry_eligible is an atomically-decidable eligibility fact committed +// WITH the fail — never a promise that a retry child exists (Elon review round 3, +// point 2). On the bulk sweeper / orphan-recovery paths the event commits inside +// FailBulkTasksWithEvents while the retry child is only created best-effort AFTER +// commit by HandleFailedTasks, where CreateRetryTask can error or the process can +// crash. We reproduce that crash window by committing the fail+event and then NOT +// running the post-commit retry step: the flag must still equal the eligibility +// predicate, the task must be terminal, and no retry child may exist — so a consumer +// can never read retry_eligible as "a fresh attempt is guaranteed to arrive". +func TestTaskFailedRetryEligibleIsNotAChildPromise(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + cases := []struct { + name string + attempt int + maxAttempts int + wantEligible bool + }{ + // A within-budget, infra-shaped failure IS eligible — yet the crash window + // means no child exists after commit until HandleFailedTasks runs. + {"within budget → eligible, still no child in the window", 1, 2, true}, + // A budget-exhausted failure is NOT eligible: the flag is decidable both + // ways, it is not hard-coded true. + {"budget exhausted → not eligible", 2, 2, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var taskID string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, attempt, max_attempts) + VALUES ($1, (SELECT runtime_id FROM agent WHERE id = $1), $2, 'running', 0, $3, $4) + RETURNING id`, agentID, issueID, tc.attempt, tc.maxAttempts).Scan(&taskID); err != nil { + t.Fatalf("seed task: %v", err) + } + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = $1 OR parent_task_id = $1`, taskID) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = $1`, taskID) + }) + + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + row, gerr := qtx.GetAgentTask(ctx, util.MustParseUUID(taskID)) + if gerr != nil { + return nil, gerr + } + return []db.AgentTaskQueue{row}, nil + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "runtime went offline", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_offline", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("FailBulkTasksWithEvents: %v", err) + } + if len(failed) != 1 { + t.Fatalf("expected 1 failed task, got %d", len(failed)) + } + + // Deliberately DO NOT call HandleFailedTasks: that post-commit step is + // where the retry child would be created. Skipping it models a crash in + // exactly that window. + + // The committed event's flag equals the eligibility predicate... + var got bool + if err := pool.QueryRow(ctx, + `SELECT (payload->>'retry_eligible')::bool FROM domain_event + WHERE subject_type = 'task' AND subject_id = $1`, taskID).Scan(&got); err != nil { + t.Fatalf("expected a task.failed domain event: %v", err) + } + if got != tc.wantEligible { + t.Errorf("retry_eligible = %v, want %v", got, tc.wantEligible) + } + // ...the subject task is terminal... + if s := taskStatusForTest(t, pool, taskID); s != "failed" { + t.Errorf("task status = %q, want failed (task.failed is terminal for the subject)", s) + } + // ...and no retry child exists — the flag is not a child promise. + var children int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue WHERE parent_task_id = $1`, taskID).Scan(&children); err != nil { + t.Fatalf("count retry children: %v", err) + } + if children != 0 { + t.Errorf("retry children = %d, want 0 (no child is created in the crash window)", children) + } + }) + } +} diff --git a/server/internal/service/recover_orphans_cursor_test.go b/server/internal/service/recover_orphans_cursor_test.go new file mode 100644 index 00000000000..49314dc71e7 --- /dev/null +++ b/server/internal/service/recover_orphans_cursor_test.go @@ -0,0 +1,134 @@ +package service + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The recover-orphans drain must step PAST a page made entirely of poison +// (unresolvable-workspace) rows to reach the healthy rows behind them (MUL-4332 +// review round 3, point 1). A poison row is skipped rather than failed, so it stays +// selectable; without the keyset cursor a plain "re-select the oldest N" drain would +// keep re-reading that same poison page forever and never fail the healthy rows +// behind it. The cursor advances over every candidate the page locked — poison +// included — so the next page moves on. +// +// A real agent_task_queue row always resolves its workspace via its agent, so (as in +// TestFailBulkTasksIsolatesPoisonRow) we present the two oldest rows to the fail path +// as poison by stripping their resolvable links in-memory; their id/created_at stay +// intact so the cursor still advances over them, and the DB rows are left untouched +// and un-failed, exactly as a genuinely unresolvable row would be. +func TestRecoverOrphansCursorStepsPastPoisonPage(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var runtimeID string + if err := pool.QueryRow(ctx, `SELECT runtime_id FROM agent WHERE id = $1`, agentID).Scan(&runtimeID); err != nil { + t.Fatalf("load runtime id: %v", err) + } + + // Three orphans on the runtime with DISTINCT, increasing created_at so ordering + // is by created_at (exercising the keyset created_at branch, complementing the + // id-tiebreaker path). The two OLDEST are the poison page; the newest is healthy. + seed := func(ageInterval string) string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, created_at) + VALUES ($1, $2, $3, 'running', 0, now() - $4::interval) + RETURNING id`, agentID, runtimeID, issueID, ageInterval).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + poison1 := seed("3 minutes") + poison2 := seed("2 minutes") + healthy := seed("1 minute") + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, []string{poison1, poison2, healthy}) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, []string{poison1, poison2, healthy}) + }) + isPoison := map[string]bool{poison1: true, poison2: true} + + // Drain exactly as the handler does, but with a page size of 2 so the two poison + // rows fill page 1 completely. If the cursor did not advance past them, page 2 + // would re-select the same poison page and the healthy row would never fail. + const pageSize = 2 + var afterCreatedAt pgtype.Timestamptz + var afterID pgtype.UUID + drained := 0 + for page := 0; page < 5; page++ { + var candidates []db.AgentTaskQueue + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: util.MustParseUUID(runtimeID), + AfterCreatedAt: afterCreatedAt, + AfterID: afterID, + MaxPerTick: pageSize, + }) + if e != nil { + return nil, e + } + // Strip links on the poison rows so the fail path cannot resolve a + // workspace and skips them (leaving them 'running'). id/created_at are + // untouched, so the cursor still advances over them. + for i := range c { + if isPoison[util.UUIDToString(c[i].ID)] { + c[i].AgentID = pgtype.UUID{} + c[i].IssueID = pgtype.UUID{} + c[i].ChatSessionID = pgtype.UUID{} + c[i].AutopilotRunID = pgtype.UUID{} + } + } + candidates = c + return c, e + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) + if err != nil { + t.Fatalf("page %d: FailBulkTasksWithEvents: %v", page, err) + } + drained += len(failed) + if len(candidates) < pageSize { + break // short page → drained + } + last := candidates[len(candidates)-1] + afterCreatedAt = last.CreatedAt + afterID = last.ID + } + + // The healthy row behind the poison page got failed with its event... + if s := taskStatusForTest(t, pool, healthy); s != "failed" { + t.Errorf("healthy row status = %q, want failed (cursor must step past the poison page)", s) + } + if n := subjectEventCount(t, pool, healthy); n != 1 { + t.Errorf("healthy row events = %d, want 1", n) + } + // ...and the poison rows were left untouched — skipped, never failed, no event. + for _, pid := range []string{poison1, poison2} { + if s := taskStatusForTest(t, pool, pid); s != "running" { + t.Errorf("poison row %s status = %q, want running (skipped, not failed)", pid, s) + } + if n := subjectEventCount(t, pool, pid); n != 0 { + t.Errorf("poison row %s events = %d, want 0", pid, n) + } + } + if drained != 1 { + t.Errorf("drained = %d, want 1 (only the healthy row behind the poison page)", drained) + } +} diff --git a/server/internal/service/task.go b/server/internal/service/task.go index d451e27df9f..af5f73b7b83 100644 --- a/server/internal/service/task.go +++ b/server/internal/service/task.go @@ -2975,19 +2975,21 @@ func (s *TaskService) FailTask(ctx context.Context, taskID pgtype.UUID, errMsg, } // Transactional outbox (MUL-4332): emit task.failed atomically with the - // fail (and any retry child). retryable reflects whether this failure - // spawned an auto-retry; the status CAS makes it exactly-once. Fail-closed - // on workspace resolution (review point 4). + // fail (and any retry child). retry_eligible is the atomically-decidable + // eligibility predicate — NOT a promise a child was created — computed from + // the just-failed row so it matches the bulk paths exactly; here a child is + // in fact created in this same tx when eligible. The status CAS makes it + // exactly-once. Fail-closed on workspace resolution (review point 4). wsID := s.resolveTaskWorkspaceForEvent(ctx, qtx, t) if !wsID.Valid { return fmt.Errorf("task.failed event: unresolvable workspace for task %s", util.UUIDToString(t.ID)) } evt := domainevent.TaskFailed(wsID, t.ID, domainevent.AgentActor(t.AgentID), domainevent.TaskFailedPayload{ - IssueID: util.UUIDToString(t.IssueID), - AgentID: util.UUIDToString(t.AgentID), - Retryable: wantRetry, - ErrorCode: failureReason, + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + RetryEligible: retryEligible(failureReason, t), + ErrorCode: failureReason, }) if _, err := domainevent.Write(ctx, qtx, evt); err != nil { return fmt.Errorf("write task.failed event: %w", err) @@ -3620,10 +3622,11 @@ func (s *TaskService) resolveTaskWorkspaceForEvent(ctx context.Context, qtx *db. // // The actor is the platform (SystemActor): these are sweeper / orphan-recovery // paths, not an agent action — the agent is already carried in the payload -// (review point 3). retryable is computed with the SAME retryEligible predicate -// HandleFailedTasks uses to decide the post-commit auto-retry, so the event and -// the retry decision agree instead of the event hard-coding false while a retry -// child appears anyway. +// (review point 3). retry_eligible is the same atomically-decidable predicate +// HandleFailedTasks uses to gate the post-commit auto-retry, committed in this +// transaction. It reports eligibility only: on these paths the retry child is +// created best-effort AFTER commit, so the event never promises a fresh attempt +// will actually arrive (review point 3, third round). // // A transient event/commit failure rolls back only the CURRENT batch (no fact is // ever committed without its event), so the next sweep tick simply re-selects and @@ -3692,10 +3695,10 @@ func (s *TaskService) FailBulkTasksWithEvents( } evt := domainevent.TaskFailed(wsID, t.ID, domainevent.SystemActor(), domainevent.TaskFailedPayload{ - IssueID: util.UUIDToString(t.IssueID), - AgentID: util.UUIDToString(t.AgentID), - Retryable: retryEligible(reason, t), - ErrorCode: reason, + IssueID: util.UUIDToString(t.IssueID), + AgentID: util.UUIDToString(t.AgentID), + RetryEligible: retryEligible(reason, t), + ErrorCode: reason, }) if _, err := domainevent.Write(ctx, qtx, evt); err != nil { return nil, err diff --git a/server/pkg/db/generated/agent.sql.go b/server/pkg/db/generated/agent.sql.go index a3e50405f25..0517e2feef9 100644 --- a/server/pkg/db/generated/agent.sql.go +++ b/server/pkg/db/generated/agent.sql.go @@ -4561,14 +4561,21 @@ func (q *Queries) SelectExpiredQueuedTasks(ctx context.Context, arg SelectExpire const selectOrphanedTasksForRuntime = `-- name: SelectOrphanedTasksForRuntime :many SELECT id, agent_id, issue_id, status, priority, dispatched_at, started_at, completed_at, result, error, created_at, context, runtime_id, session_id, work_dir, trigger_comment_id, chat_session_id, autopilot_run_id, attempt, max_attempts, parent_task_id, failure_reason, trigger_summary, force_fresh_session, is_leader_task, wait_reason, initiator_user_id, handoff_note, prepare_lease_expires_at, squad_id, runtime_mcp_overlay, escalation_for_task_id, fire_at, originator_user_id, runtime_connected_apps, coalesced_comment_ids, delivered_comment_ids, chat_input_task_id, chat_finalize_deferred_at, originator_source, delegated_from_task_id, retry_of_task_id, rerun_of_task_id, rule_version_id, trigger_evidence_kind, trigger_evidence_ref_id, accountable_user_id FROM agent_task_queue WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory') -ORDER BY created_at ASC -LIMIT $2::int + AND ( + $2::timestamptz IS NULL + OR created_at > $2::timestamptz + OR (created_at = $2::timestamptz AND id > $3::uuid) + ) +ORDER BY created_at ASC, id ASC +LIMIT $4::int FOR UPDATE SKIP LOCKED ` type SelectOrphanedTasksForRuntimeParams struct { - RuntimeID pgtype.UUID `json:"runtime_id"` - MaxPerTick int32 `json:"max_per_tick"` + RuntimeID pgtype.UUID `json:"runtime_id"` + AfterCreatedAt pgtype.Timestamptz `json:"after_created_at"` + AfterID pgtype.UUID `json:"after_id"` + MaxPerTick int32 `json:"max_per_tick"` } // Called by the daemon at startup. Selects (and row-locks) up to @max_per_tick @@ -4582,8 +4589,24 @@ type SelectOrphanedTasksForRuntimeParams struct { // FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. // FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's // own claim path; the LIMIT bounds the lock hold. +// +// Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts +// the runtime back to `online`, so the every-tick offline sweep will NOT reap an +// orphan the daemon leaves past this page — the recovery must therefore drain +// itself. Callers page by (created_at, id) via @after_created_at / @after_id (NULL +// on the first page) and repeat until a short page. Because the cursor advances +// over EVERY returned candidate — including a poison (unresolvable-workspace) row +// the caller skips rather than fails — a page full of poison at the front can no +// longer pin the drain in place: the next page steps past it to the healthy rows +// behind it. (A plain re-select of the oldest rows would loop on the poison +// forever.) Ordering matches the keyset so pages are stable and non-overlapping. func (q *Queries) SelectOrphanedTasksForRuntime(ctx context.Context, arg SelectOrphanedTasksForRuntimeParams) ([]AgentTaskQueue, error) { - rows, err := q.db.Query(ctx, selectOrphanedTasksForRuntime, arg.RuntimeID, arg.MaxPerTick) + rows, err := q.db.Query(ctx, selectOrphanedTasksForRuntime, + arg.RuntimeID, + arg.AfterCreatedAt, + arg.AfterID, + arg.MaxPerTick, + ) if err != nil { return nil, err } diff --git a/server/pkg/db/queries/agent.sql b/server/pkg/db/queries/agent.sql index 53c27a0dd3a..ec1dbeb591f 100644 --- a/server/pkg/db/queries/agent.sql +++ b/server/pkg/db/queries/agent.sql @@ -710,9 +710,25 @@ WHERE id = $1 AND status IN ('dispatched', 'running'); -- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. -- FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's -- own claim path; the LIMIT bounds the lock hold. +-- +-- Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts +-- the runtime back to `online`, so the every-tick offline sweep will NOT reap an +-- orphan the daemon leaves past this page — the recovery must therefore drain +-- itself. Callers page by (created_at, id) via @after_created_at / @after_id (NULL +-- on the first page) and repeat until a short page. Because the cursor advances +-- over EVERY returned candidate — including a poison (unresolvable-workspace) row +-- the caller skips rather than fails — a page full of poison at the front can no +-- longer pin the drain in place: the next page steps past it to the healthy rows +-- behind it. (A plain re-select of the oldest rows would loop on the poison +-- forever.) Ordering matches the keyset so pages are stable and non-overlapping. SELECT * FROM agent_task_queue WHERE runtime_id = @runtime_id AND status IN ('dispatched', 'running', 'waiting_local_directory') -ORDER BY created_at ASC + AND ( + sqlc.narg('after_created_at')::timestamptz IS NULL + OR created_at > sqlc.narg('after_created_at')::timestamptz + OR (created_at = sqlc.narg('after_created_at')::timestamptz AND id > sqlc.narg('after_id')::uuid) + ) +ORDER BY created_at ASC, id ASC LIMIT @max_per_tick::int FOR UPDATE SKIP LOCKED; From 50e9895a8886e07a195cce45fce35de99ed1d4be Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 15:07:30 +0800 Subject: [PATCH 05/15] fix(automation): recover-orphans lock-wait + legacy-daemon drain (MUL-4332 PR1) Address Elon's round-4 review on PR #5467. 1. recover-orphans keyset cursor + SKIP LOCKED race (point 1): SelectOrphanedTasksForRuntime now uses plain FOR UPDATE. The drain pages forward with a keyset cursor that advances permanently, so SKIP LOCKED could silently drop a briefly-locked older orphan and step the cursor past it, leaking it forever (the re-registered runtime is already online, so the offline sweep won't reap it). Plain FOR UPDATE waits for the lock and re-checks status, so the older row is included. Only this query changes; the sweepers re-scan every tick and keep SKIP LOCKED. 2. Legacy daemon {} backward compat (point 2): A current daemon now sends paginate:true and drives the drain itself. A legacy daemon POSTs {} once and ignores the body, so the server drains every page itself in bounded per-page transactions instead of leaving 501+ orphans stranded past the first page. Both modes share recoverOrphansPage. Tests: TestRecoverOrphansWaitsForLockedOldestOrphan (blocks on a locked oldest orphan, then fails it with its event once released), and TestRecoverOrphansLegacyClientDrainsServerSide (501 rows, one {} call, all failed + events). The existing paginated drain test now sends paginate:true. Rebased onto main (inbox index migrations landed at 200/201); renumbered the six domain_event migrations 200-205 -> 202-207 and fixed their cross-references. Co-authored-by: multica-agent --- server/internal/daemon/client.go | 8 +- .../handler/recover_orphans_drain_test.go | 92 +++++++++- server/internal/handler/task_lifecycle.go | 107 +++++++++-- .../service/recover_orphans_lock_wait_test.go | 168 ++++++++++++++++++ ...ent.down.sql => 202_domain_event.down.sql} | 2 +- ...n_event.up.sql => 202_domain_event.up.sql} | 2 +- ...03_domain_event_seq_unique_index.down.sql} | 0 ... 203_domain_event_seq_unique_index.up.sql} | 0 ... 204_domain_event_dispatch_index.down.sql} | 0 ...=> 204_domain_event_dispatch_index.up.sql} | 2 +- ...5_domain_event_correlation_index.down.sql} | 0 ...205_domain_event_correlation_index.up.sql} | 2 +- ...l => 206_domain_event_type_index.down.sql} | 0 ...sql => 206_domain_event_type_index.up.sql} | 2 +- ...> 207_domain_event_subject_index.down.sql} | 0 ... => 207_domain_event_subject_index.up.sql} | 2 +- server/pkg/db/generated/agent.sql.go | 20 ++- server/pkg/db/queries/agent.sql | 20 ++- 18 files changed, 397 insertions(+), 30 deletions(-) create mode 100644 server/internal/service/recover_orphans_lock_wait_test.go rename server/migrations/{200_domain_event.down.sql => 202_domain_event.down.sql} (75%) rename server/migrations/{200_domain_event.up.sql => 202_domain_event.up.sql} (98%) rename server/migrations/{201_domain_event_seq_unique_index.down.sql => 203_domain_event_seq_unique_index.down.sql} (100%) rename server/migrations/{201_domain_event_seq_unique_index.up.sql => 203_domain_event_seq_unique_index.up.sql} (100%) rename server/migrations/{202_domain_event_dispatch_index.down.sql => 204_domain_event_dispatch_index.down.sql} (100%) rename server/migrations/{202_domain_event_dispatch_index.up.sql => 204_domain_event_dispatch_index.up.sql} (81%) rename server/migrations/{203_domain_event_correlation_index.down.sql => 205_domain_event_correlation_index.down.sql} (100%) rename server/migrations/{203_domain_event_correlation_index.up.sql => 205_domain_event_correlation_index.up.sql} (81%) rename server/migrations/{204_domain_event_type_index.down.sql => 206_domain_event_type_index.down.sql} (100%) rename server/migrations/{204_domain_event_type_index.up.sql => 206_domain_event_type_index.up.sql} (78%) rename server/migrations/{205_domain_event_subject_index.down.sql => 207_domain_event_subject_index.down.sql} (100%) rename server/migrations/{205_domain_event_subject_index.up.sql => 207_domain_event_subject_index.up.sql} (81%) diff --git a/server/internal/daemon/client.go b/server/internal/daemon/client.go index b1f960f9716..f14ca179722 100644 --- a/server/internal/daemon/client.go +++ b/server/internal/daemon/client.go @@ -439,7 +439,12 @@ const maxRecoverOrphanPages = 1000 // server skips, so a page of poison at the front cannot stall the drain. func (c *Client) RecoverOrphans(ctx context.Context, runtimeID string) error { path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) - body := map[string]any{} + // paginate:true is our capability signal (MUL-4332 review round 4, point 2): it + // tells the server this client will drive the drain itself, so the server hands + // back one page at a time. A server that predates the flag ignores it and does + // its own single-shot recovery (handled via io.EOF below); a legacy daemon that + // lacks the flag is drained entirely server-side. + body := map[string]any{"paginate": true} for page := 0; page < maxRecoverOrphanPages; page++ { var resp struct { HasMore bool `json:"has_more"` @@ -461,6 +466,7 @@ func (c *Client) RecoverOrphans(ctx context.Context, runtimeID string) error { return nil } body = map[string]any{ + "paginate": true, "cursor_created_at": resp.NextCursorCreatedAt, "cursor_id": resp.NextCursorID, } diff --git a/server/internal/handler/recover_orphans_drain_test.go b/server/internal/handler/recover_orphans_drain_test.go index a7625a7daaa..e2605d9ac9a 100644 --- a/server/internal/handler/recover_orphans_drain_test.go +++ b/server/internal/handler/recover_orphans_drain_test.go @@ -61,9 +61,12 @@ func TestRecoverOrphansDrainsPastBatchLimit(t *testing.T) { } // Drive the real handler in a drain loop, threading the cursor like the daemon. + // paginate:true is the current daemon's capability signal — without it the server + // treats the caller as a legacy client and drains every page itself in one call + // (that legacy path is covered by TestRecoverOrphansLegacyClientDrainsServerSide). const daemonID = "recover-drain-test" path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) - var body any + var body any = map[string]any{"paginate": true} totalFailed, pages := 0, 0 for { pages++ @@ -87,7 +90,7 @@ func TestRecoverOrphansDrainsPastBatchLimit(t *testing.T) { if resp.NextCursorCreatedAt == "" || resp.NextCursorID == "" { t.Fatalf("page %d: has_more=true but empty cursor", pages) } - body = map[string]any{"cursor_created_at": resp.NextCursorCreatedAt, "cursor_id": resp.NextCursorID} + body = map[string]any{"paginate": true, "cursor_created_at": resp.NextCursorCreatedAt, "cursor_id": resp.NextCursorID} } if pages != 2 { @@ -119,3 +122,88 @@ func TestRecoverOrphansDrainsPastBatchLimit(t *testing.T) { t.Errorf("task.failed events = %d, want %d (fact ⇔ event, one per failed row)", events, total) } } + +// A LEGACY daemon (one that predates paging) POSTs {} exactly once and ignores the +// response body, so it can never thread the keyset cursor across pages. With 501+ +// orphans on a re-registered runtime — which registration has already flipped back +// `online`, so the offline sweep won't reap the tail — the server must therefore +// drain every page itself in one call (MUL-4332 review round 4, point 2). We seed +// orphanRecoveryBatchSize + 1 orphans, POST a single {} request with NO paginate +// capability, and assert all of them end failed with one event each — none leak past +// the first server page. +func TestRecoverOrphansLegacyClientDrainsServerSide(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + + var runtimeID, agentID string + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent_runtime (workspace_id, name, runtime_mode, provider, status, device_info, metadata, owner_id) + VALUES ($1, 'orphan-legacy-runtime', 'cloud', 'codex', 'online', '', '{}'::jsonb, $2) + RETURNING id`, testWorkspaceID, testUserID).Scan(&runtimeID); err != nil { + t.Fatalf("seed runtime: %v", err) + } + if err := testPool.QueryRow(ctx, ` + INSERT INTO agent (workspace_id, name, runtime_mode, runtime_config, runtime_id, visibility, + max_concurrent_tasks, owner_id, instructions, custom_env, custom_args) + VALUES ($1, 'orphan-legacy-agent', 'cloud', '{}'::jsonb, $2, 'workspace', 1, $3, '', '{}'::jsonb, '[]'::jsonb) + RETURNING id`, testWorkspaceID, runtimeID, testUserID).Scan(&agentID); err != nil { + t.Fatalf("seed agent: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), + `DELETE FROM domain_event WHERE subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, runtimeID) + testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID) + }) + + // One past a single server page (see TestRecoverOrphansDrainsPastBatchLimit for + // why attempt/max with no issue keeps the post-fail pipeline a no-op here). + total := orphanRecoveryBatchSize + 1 + if _, err := testPool.Exec(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, status, priority) + SELECT $1, $2, 'running', 0 FROM generate_series(1, $3)`, agentID, runtimeID, total); err != nil { + t.Fatalf("seed orphaned tasks: %v", err) + } + + // Legacy request: {} body, no paginate capability, called exactly once. + const daemonID = "recover-legacy-test" + path := fmt.Sprintf("/api/daemon/runtimes/%s/recover-orphans", runtimeID) + w := httptest.NewRecorder() + req := withURLParam(newDaemonTokenRequest(http.MethodPost, path, map[string]any{}, testWorkspaceID, daemonID), "runtimeId", runtimeID) + testHandler.RecoverOrphanedTasks(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var resp RecoverOrphansResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.HasMore { + t.Errorf("legacy drain returned has_more=true; the server must fully drain in one call (a legacy client cannot page)") + } + if resp.Orphaned != total { + t.Errorf("orphaned = %d, want %d (single legacy call must drain past the first server page)", resp.Orphaned, total) + } + + var remaining int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM agent_task_queue + WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_directory')`, + runtimeID).Scan(&remaining); err != nil { + t.Fatalf("count remaining: %v", err) + } + if remaining != 0 { + t.Errorf("remaining orphans = %d, want 0 (legacy client can't page, so nothing may be left behind)", remaining) + } + var events int + if err := testPool.QueryRow(ctx, + `SELECT count(*) FROM domain_event + WHERE type = 'task.failed' AND subject_id IN (SELECT id FROM agent_task_queue WHERE runtime_id = $1)`, + runtimeID).Scan(&events); err != nil { + t.Fatalf("count events: %v", err) + } + if events != total { + t.Errorf("task.failed events = %d, want %d (fact ⇔ event, one per failed row)", events, total) + } +} diff --git a/server/internal/handler/task_lifecycle.go b/server/internal/handler/task_lifecycle.go index 0bd835a6b9e..4e1ed6abd66 100644 --- a/server/internal/handler/task_lifecycle.go +++ b/server/internal/handler/task_lifecycle.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "errors" "io" @@ -22,11 +23,27 @@ import ( // lock hold and event volume per page. const orphanRecoveryBatchSize = 500 -// RecoverOrphansRequest is the optional body of POST recover-orphans. Empty on the -// first page; the daemon threads back the keyset cursor from the prior page's -// response to fetch the next one. Both fields move together — a partial cursor is -// rejected. +// maxServerOrphanDrainPages bounds the server-side drain loop used for legacy +// clients (see RecoverOrphanedTasks). Each page fails up to orphanRecoveryBatchSize +// (500) rows, so this covers ~500k orphaned rows in one request — far beyond any +// real restart. It is purely a backstop so a pathological backlog can never hold +// one HTTP request open forever; any residual is reclaimed on the next +// registration. It mirrors the daemon client's own maxRecoverOrphanPages backstop. +const maxServerOrphanDrainPages = 1000 + +// RecoverOrphansRequest is the optional body of POST recover-orphans. +// +// Paginate is the client's capability signal (MUL-4332 review round 4, point 2): +// a current daemon sets it true and drives the drain itself, threading the keyset +// cursor across calls. A legacy daemon predates paging — it POSTs `{}` (or an empty +// body) exactly once and ignores the response — so Paginate is absent/false and the +// server must instead drain every page itself, or orphans past the first page leak +// on a re-registered (now `online`) runtime that the offline sweep won't reap. +// +// Cursor* is threaded back by a paginating client for each subsequent page. Both +// fields move together — a partial cursor is rejected. type RecoverOrphansRequest struct { + Paginate bool `json:"paginate,omitempty"` CursorCreatedAt string `json:"cursor_created_at,omitempty"` CursorID string `json:"cursor_id,omitempty"` } @@ -78,16 +95,74 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { return } + runtimeUUID := parseUUID(runtimeID) + + // A capability-aware client (req.Paginate) drives the drain itself: fail exactly + // one page and hand back the cursor for it to continue. A legacy client — which + // POSTs {} once and ignores the body — cannot page, so the server drains every + // page itself in bounded per-page transactions; otherwise 501+ orphans on a + // re-registered (now `online`) runtime leak permanently (MUL-4332 review round 4, + // point 2). Both modes share recoverOrphansPage, so the per-page fact ⇔ event and + // poison-isolation semantics are identical. + if req.Paginate { + resp, _, _, err := h.recoverOrphansPage(r.Context(), runtimeUUID, runtimeID, afterCreatedAt, afterID) + if err != nil { + slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) + writeError(w, http.StatusInternalServerError, "recover orphans failed") + return + } + writeJSON(w, http.StatusOK, resp) + return + } + + // Legacy single-shot client: drain all pages server-side. A partial cursor is + // already rejected above, so a legacy client always starts from the first page + // (both cursor components NULL). + var agg RecoverOrphansResponse + cursorCreatedAt, cursorID := afterCreatedAt, afterID + for page := 0; page < maxServerOrphanDrainPages; page++ { + resp, nextCreatedAt, nextID, err := h.recoverOrphansPage(r.Context(), runtimeUUID, runtimeID, cursorCreatedAt, cursorID) + if err != nil { + slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) + writeError(w, http.StatusInternalServerError, "recover orphans failed") + return + } + agg.Orphaned += resp.Orphaned + agg.Retried += resp.Retried + agg.Skipped += resp.Skipped + if !resp.HasMore { + break + } + cursorCreatedAt, cursorID = nextCreatedAt, nextID + } + // The legacy client ignores the body, but return an accurate, explicitly + // non-paged (has_more=false) aggregate for logs and proxies. + writeJSON(w, http.StatusOK, agg) +} + +// recoverOrphansPage fails one bounded page of the runtime's orphaned tasks and +// runs the shared post-failure pipeline. It returns the page result plus the raw +// keyset cursor of the last candidate (for the server-side legacy drain to thread +// without re-parsing its own RFC3339 string). The cursor advances over EVERY +// candidate the page locked — failed OR skipped-poison — so neither a paginating +// client nor the server-side drain can be pinned by a page of poison at the front. +func (h *Handler) recoverOrphansPage( + ctx context.Context, + runtimeUUID pgtype.UUID, + runtimeID string, + afterCreatedAt pgtype.Timestamptz, + afterID pgtype.UUID, +) (RecoverOrphansResponse, pgtype.Timestamptz, pgtype.UUID, error) { // Select one page of the runtime's orphaned tasks and fail the resolvable ones // with their task.failed events atomically (MUL-4332 review point 2), so - // daemon-driven recovery converges onto the outbox like the runtime sweepers and - // an unresolvable poison row cannot block the rest. We capture the raw candidate + // recovery converges onto the outbox like the runtime sweepers and an + // unresolvable poison row cannot block the rest. We capture the raw candidate // page (before poison isolation drops rows) to compute has_more and the cursor. var candidates []db.AgentTaskQueue - rows, err := h.TaskService.FailBulkTasksWithEvents(r.Context(), + rows, err := h.TaskService.FailBulkTasksWithEvents(ctx, func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { - c, e := qtx.SelectOrphanedTasksForRuntime(r.Context(), db.SelectOrphanedTasksForRuntimeParams{ - RuntimeID: parseUUID(runtimeID), + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: runtimeUUID, AfterCreatedAt: afterCreatedAt, AfterID: afterID, MaxPerTick: orphanRecoveryBatchSize, @@ -96,16 +171,14 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { return c, e }, func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { - return qtx.FailAgentTasksByIDs(r.Context(), db.FailAgentTasksByIDsParams{ + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ Ids: ids, Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, }) }) if err != nil { - slog.Warn("recover-orphans failed", "runtime_id", runtimeID, "error", err) - writeError(w, http.StatusInternalServerError, "recover orphans failed") - return + return RecoverOrphansResponse{}, pgtype.Timestamptz{}, pgtype.UUID{}, err } // Funnel through the shared post-failure pipeline so we get the same @@ -113,7 +186,7 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { // behaviour as the runtime sweeper. This was previously a fast-path // that bypassed those side effects, leaving the UI stale when no retry // was created (max_attempts exhausted, autopilot, non-retryable reason). - retried := h.TaskService.HandleFailedTasks(r.Context(), rows) + retried := h.TaskService.HandleFailedTasks(ctx, rows) // A full candidate page means there may be more behind it. The cursor advances // over the LAST candidate we locked (failed or skipped), so the next page never @@ -124,8 +197,12 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { Skipped: len(candidates) - len(rows), HasMore: len(candidates) == orphanRecoveryBatchSize, } + var nextCreatedAt pgtype.Timestamptz + var nextID pgtype.UUID if resp.HasMore && len(candidates) > 0 { last := candidates[len(candidates)-1] + nextCreatedAt = last.CreatedAt + nextID = last.ID resp.NextCursorCreatedAt = last.CreatedAt.Time.UTC().Format(time.RFC3339Nano) resp.NextCursorID = uuidToString(last.ID) } @@ -141,7 +218,7 @@ func (h *Handler) RecoverOrphanedTasks(w http.ResponseWriter, r *http.Request) { ) } - writeJSON(w, http.StatusOK, resp) + return resp, nextCreatedAt, nextID, nil } // parseOrphanCursor turns the optional (cursor_created_at, cursor_id) request pair diff --git a/server/internal/service/recover_orphans_lock_wait_test.go b/server/internal/service/recover_orphans_lock_wait_test.go new file mode 100644 index 00000000000..4ed9719019d --- /dev/null +++ b/server/internal/service/recover_orphans_lock_wait_test.go @@ -0,0 +1,168 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/events" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The recover-orphans query pages forward with a keyset cursor that advances +// permanently past whatever a page returned, so it MUST NOT use FOR UPDATE SKIP +// LOCKED (MUL-4332 review round 4, point 1). If an older orphan is briefly locked +// while a page is being selected — a sweep or a stale-dispatch reclaim holding it — +// SKIP LOCKED would silently drop it; the page, filled by the newer rows behind it, +// would advance the cursor past that older row, and no later page could ever select +// it again (the runtime is already back `online`, so the offline sweep won't reap it +// either). Plain FOR UPDATE instead WAITS for the lock and, once it releases, +// re-checks the row against the WHERE and includes it. +// +// This reproduces exactly that race: lock the OLDEST orphan in a concurrent +// transaction, prove the drain BLOCKS (does not skip past) while the lock is held, +// then release it and assert the previously-locked row is failed with its event. +// Under the old SKIP LOCKED query the drain would finish immediately with the oldest +// row still `running` forever, and this test would fail. +func TestRecoverOrphansWaitsForLockedOldestOrphan(t *testing.T) { + pool := newTaskClaimRacePool(t) // skips if no DB + ctx := context.Background() + queries := db.New(pool) + svc := NewTaskService(queries, pool, nil, events.New()) + + _, _, agentID, issueID := seedAttributionFixture(t, pool) + + var runtimeID string + if err := pool.QueryRow(ctx, `SELECT runtime_id FROM agent WHERE id = $1`, agentID).Scan(&runtimeID); err != nil { + t.Fatalf("load runtime id: %v", err) + } + + // Three healthy orphans with distinct, increasing created_at. The OLDEST is the + // one we lock while the page is selected. + seed := func(ageInterval string) string { + var id string + if err := pool.QueryRow(ctx, ` + INSERT INTO agent_task_queue (agent_id, runtime_id, issue_id, status, priority, created_at) + VALUES ($1, $2, $3, 'running', 0, now() - $4::interval) + RETURNING id`, agentID, runtimeID, issueID, ageInterval).Scan(&id); err != nil { + t.Fatalf("seed task: %v", err) + } + return id + } + oldest := seed("3 minutes") + mid := seed("2 minutes") + newest := seed("1 minute") + all := []string{oldest, mid, newest} + t.Cleanup(func() { + pool.Exec(context.Background(), `DELETE FROM agent_task_queue WHERE id = ANY($1::uuid[])`, all) + pool.Exec(context.Background(), `DELETE FROM domain_event WHERE subject_id = ANY($1::uuid[])`, all) + }) + + // Hold a FOR UPDATE lock on the oldest orphan in a dedicated connection, exactly + // as a sweep or a stale-dispatch reclaim would briefly hold it. + lockConn, err := pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire lock conn: %v", err) + } + lockReleased := false + releaseLock := func() { + if lockReleased { + return + } + lockReleased = true + lockConn.Release() + } + defer releaseLock() + + lockTx, err := lockConn.Begin(ctx) + if err != nil { + t.Fatalf("begin lock tx: %v", err) + } + if _, err := lockTx.Exec(ctx, `SELECT id FROM agent_task_queue WHERE id = $1 FOR UPDATE`, oldest); err != nil { + t.Fatalf("lock oldest orphan: %v", err) + } + + // Run one drain page in the background. A page size above the row count means a + // single page covers all three — so blocking on the oldest row blocks the whole + // page (nothing is partially failed until the lock releases). + type drainResult struct { + failed int + err error + } + done := make(chan drainResult, 1) + go func() { + var candidates []db.AgentTaskQueue + failed, err := svc.FailBulkTasksWithEvents(ctx, + func(qtx *db.Queries) ([]db.AgentTaskQueue, error) { + c, e := qtx.SelectOrphanedTasksForRuntime(ctx, db.SelectOrphanedTasksForRuntimeParams{ + RuntimeID: util.MustParseUUID(runtimeID), + MaxPerTick: 500, + }) + candidates = c + return c, e + }, + func(qtx *db.Queries, ids []pgtype.UUID) ([]db.AgentTaskQueue, error) { + return qtx.FailAgentTasksByIDs(ctx, db.FailAgentTasksByIDsParams{ + Ids: ids, + Error: pgtype.Text{String: "daemon restarted while task was in flight", Valid: true}, + FailureReason: pgtype.Text{String: "runtime_recovery", Valid: true}, + }) + }) + _ = candidates + done <- drainResult{failed: len(failed), err: err} + }() + + // While the lock is held the drain must NOT finish: plain FOR UPDATE blocks on + // the locked oldest row instead of skipping it. + select { + case res := <-done: + if res.err != nil { + t.Fatalf("drain errored while the oldest orphan was locked: %v", res.err) + } + t.Fatalf("drain completed (%d failed) while the oldest orphan was locked — it skipped instead of waiting (SKIP LOCKED regression)", res.failed) + case <-time.After(750 * time.Millisecond): + // Expected: blocked on the lock. + } + + // Nothing may be partially failed while the page is blocked behind the lock. + for _, id := range all { + if s := taskStatusForTest(t, pool, id); s != "running" { + t.Fatalf("row %s = %q before unlock, want running (a blocked page must fail nothing yet)", id, s) + } + } + + // Release the lock; the drain must now unblock and fail every orphan. + if err := lockTx.Rollback(ctx); err != nil { + t.Fatalf("release lock: %v", err) + } + releaseLock() + + select { + case res := <-done: + if res.err != nil { + t.Fatalf("drain errored after the lock released: %v", res.err) + } + if res.failed != len(all) { + t.Errorf("failed = %d, want %d (the drain must include the previously-locked oldest row)", res.failed, len(all)) + } + case <-time.After(5 * time.Second): + t.Fatal("drain did not complete after the lock released — plain FOR UPDATE never unblocked") + } + + // The previously-locked oldest orphan is now failed with exactly one event. + if s := taskStatusForTest(t, pool, oldest); s != "failed" { + t.Errorf("oldest orphan status = %q, want failed (plain FOR UPDATE must wait for the lock, not skip past)", s) + } + if n := subjectEventCount(t, pool, oldest); n != 1 { + t.Errorf("oldest orphan events = %d, want 1 (fact ⇔ event for the recovered row)", n) + } + // The other two rows are failed too, so the whole page landed. + for _, id := range []string{mid, newest} { + if s := taskStatusForTest(t, pool, id); s != "failed" { + t.Errorf("row %s status = %q, want failed", id, s) + } + } +} diff --git a/server/migrations/200_domain_event.down.sql b/server/migrations/202_domain_event.down.sql similarity index 75% rename from server/migrations/200_domain_event.down.sql rename to server/migrations/202_domain_event.down.sql index 7b672f13edb..d41a8615784 100644 --- a/server/migrations/200_domain_event.down.sql +++ b/server/migrations/202_domain_event.down.sql @@ -1,4 +1,4 @@ --- Reverses 200_domain_event.up.sql. Dropping the table also drops the +-- Reverses 202_domain_event.up.sql. Dropping the table also drops the -- sequence it OWNS; the explicit DROP SEQUENCE is a defensive no-op in case -- the ownership link is ever severed. DROP TABLE IF EXISTS domain_event; diff --git a/server/migrations/200_domain_event.up.sql b/server/migrations/202_domain_event.up.sql similarity index 98% rename from server/migrations/200_domain_event.up.sql rename to server/migrations/202_domain_event.up.sql index 3dbf9d3a3a1..2ce6dfa7420 100644 --- a/server/migrations/200_domain_event.up.sql +++ b/server/migrations/202_domain_event.up.sql @@ -20,7 +20,7 @@ -- Workspace DB rules (CLAUDE.md + MUL-4332 §4): NO foreign key, NO cascade — -- every UUID association is validated in the application layer. Secondary and -- unique indexes are added in their own single-statement CONCURRENTLY --- migrations (201–205), never inline, so index builds never take an ACCESS +-- migrations (203–207), never inline, so index builds never take an ACCESS -- EXCLUSIVE lock during deploy. -- Monotonic dispatch / drain boundary. `seq` orders events for stable scanning diff --git a/server/migrations/201_domain_event_seq_unique_index.down.sql b/server/migrations/203_domain_event_seq_unique_index.down.sql similarity index 100% rename from server/migrations/201_domain_event_seq_unique_index.down.sql rename to server/migrations/203_domain_event_seq_unique_index.down.sql diff --git a/server/migrations/201_domain_event_seq_unique_index.up.sql b/server/migrations/203_domain_event_seq_unique_index.up.sql similarity index 100% rename from server/migrations/201_domain_event_seq_unique_index.up.sql rename to server/migrations/203_domain_event_seq_unique_index.up.sql diff --git a/server/migrations/202_domain_event_dispatch_index.down.sql b/server/migrations/204_domain_event_dispatch_index.down.sql similarity index 100% rename from server/migrations/202_domain_event_dispatch_index.down.sql rename to server/migrations/204_domain_event_dispatch_index.down.sql diff --git a/server/migrations/202_domain_event_dispatch_index.up.sql b/server/migrations/204_domain_event_dispatch_index.up.sql similarity index 81% rename from server/migrations/202_domain_event_dispatch_index.up.sql rename to server/migrations/204_domain_event_dispatch_index.up.sql index 45738132b2a..ecb65e3f452 100644 --- a/server/migrations/202_domain_event_dispatch_index.up.sql +++ b/server/migrations/204_domain_event_dispatch_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 201). Backs the PR3 matcher's +-- Single-statement CONCURRENTLY migration (see 203). Backs the PR3 matcher's -- claim scan for undispatched, now-available events in seq order: -- WHERE dispatch_status = 'pending' AND available_at <= now() ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_dispatch diff --git a/server/migrations/203_domain_event_correlation_index.down.sql b/server/migrations/205_domain_event_correlation_index.down.sql similarity index 100% rename from server/migrations/203_domain_event_correlation_index.down.sql rename to server/migrations/205_domain_event_correlation_index.down.sql diff --git a/server/migrations/203_domain_event_correlation_index.up.sql b/server/migrations/205_domain_event_correlation_index.up.sql similarity index 81% rename from server/migrations/203_domain_event_correlation_index.up.sql rename to server/migrations/205_domain_event_correlation_index.up.sql index 525149ebcc0..eb8290a9528 100644 --- a/server/migrations/203_domain_event_correlation_index.up.sql +++ b/server/migrations/205_domain_event_correlation_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 201). Backs correlation-chain +-- Single-statement CONCURRENTLY migration (see 203). Backs correlation-chain -- reads (GET /api/events?correlation_id=) and loop/depth guardrail lookups: -- WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_correlation diff --git a/server/migrations/204_domain_event_type_index.down.sql b/server/migrations/206_domain_event_type_index.down.sql similarity index 100% rename from server/migrations/204_domain_event_type_index.down.sql rename to server/migrations/206_domain_event_type_index.down.sql diff --git a/server/migrations/204_domain_event_type_index.up.sql b/server/migrations/206_domain_event_type_index.up.sql similarity index 78% rename from server/migrations/204_domain_event_type_index.up.sql rename to server/migrations/206_domain_event_type_index.up.sql index 6deea6519e8..3f5f380ce8e 100644 --- a/server/migrations/204_domain_event_type_index.up.sql +++ b/server/migrations/206_domain_event_type_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 201). Backs per-workspace, +-- Single-statement CONCURRENTLY migration (see 203). Backs per-workspace, -- per-type event scans used by explain/debug tooling: -- WHERE workspace_id = $1 AND type = $2 ORDER BY seq CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_domain_event_type diff --git a/server/migrations/205_domain_event_subject_index.down.sql b/server/migrations/207_domain_event_subject_index.down.sql similarity index 100% rename from server/migrations/205_domain_event_subject_index.down.sql rename to server/migrations/207_domain_event_subject_index.down.sql diff --git a/server/migrations/205_domain_event_subject_index.up.sql b/server/migrations/207_domain_event_subject_index.up.sql similarity index 81% rename from server/migrations/205_domain_event_subject_index.up.sql rename to server/migrations/207_domain_event_subject_index.up.sql index b4f36da6f57..68d78abc4e9 100644 --- a/server/migrations/205_domain_event_subject_index.up.sql +++ b/server/migrations/207_domain_event_subject_index.up.sql @@ -1,4 +1,4 @@ --- Single-statement CONCURRENTLY migration (see 201). Backs "all events about +-- Single-statement CONCURRENTLY migration (see 203). Backs "all events about -- this subject" scans (a given issue / comment / task) for debug + the future -- stage sensor: -- WHERE subject_type = $1 AND subject_id = $2 ORDER BY seq diff --git a/server/pkg/db/generated/agent.sql.go b/server/pkg/db/generated/agent.sql.go index 0517e2feef9..51de3c555b6 100644 --- a/server/pkg/db/generated/agent.sql.go +++ b/server/pkg/db/generated/agent.sql.go @@ -4568,7 +4568,7 @@ WHERE runtime_id = $1 AND status IN ('dispatched', 'running', 'waiting_local_dir ) ORDER BY created_at ASC, id ASC LIMIT $4::int -FOR UPDATE SKIP LOCKED +FOR UPDATE ` type SelectOrphanedTasksForRuntimeParams struct { @@ -4587,8 +4587,22 @@ type SelectOrphanedTasksForRuntimeParams struct { // Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the // caller resolves each task's workspace, fails only the resolvable set via // FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. -// FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's -// own claim path; the LIMIT bounds the lock hold. +// The LIMIT bounds the lock hold. +// +// Plain FOR UPDATE (NOT SKIP LOCKED — review round 4, point 1). Unlike the +// sweepers, which re-scan from the start every tick so a skipped row is simply +// retried next tick, this query pages forward with a keyset cursor that advances +// permanently past whatever a page returned. If SKIP LOCKED silently dropped an +// older orphan that happened to be briefly locked (a sweep, a stale-dispatch +// reclaim), the cursor — filled by the newer rows behind it — would step past that +// older row and NO later page could ever select it again; the runtime is already +// back `online`, so the offline sweep won't reap it either, and it leaks forever. +// Plain FOR UPDATE instead WAITS for the lock and, once it releases, re-checks the +// row against the WHERE (Postgres EvalPlanQual): still dispatched/running → +// included on this page; already failed by whoever held the lock → correctly +// excluded. The bounded page size keeps that wait short, and recovery only races +// short sweeper/reclaim transactions (both SKIP LOCKED, so they never wait — no +// deadlock cycle is possible). // // Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts // the runtime back to `online`, so the every-tick offline sweep will NOT reap an diff --git a/server/pkg/db/queries/agent.sql b/server/pkg/db/queries/agent.sql index ec1dbeb591f..b5fd1eae25f 100644 --- a/server/pkg/db/queries/agent.sql +++ b/server/pkg/db/queries/agent.sql @@ -708,8 +708,22 @@ WHERE id = $1 AND status IN ('dispatched', 'running'); -- Candidate half of the MUL-4332 poison-isolated fail path (review point 2): the -- caller resolves each task's workspace, fails only the resolvable set via -- FailAgentTasksByIDs and emits its task.failed event in the SAME transaction. --- FOR UPDATE SKIP LOCKED avoids contending with the freshly-restarted daemon's --- own claim path; the LIMIT bounds the lock hold. +-- The LIMIT bounds the lock hold. +-- +-- Plain FOR UPDATE (NOT SKIP LOCKED — review round 4, point 1). Unlike the +-- sweepers, which re-scan from the start every tick so a skipped row is simply +-- retried next tick, this query pages forward with a keyset cursor that advances +-- permanently past whatever a page returned. If SKIP LOCKED silently dropped an +-- older orphan that happened to be briefly locked (a sweep, a stale-dispatch +-- reclaim), the cursor — filled by the newer rows behind it — would step past that +-- older row and NO later page could ever select it again; the runtime is already +-- back `online`, so the offline sweep won't reap it either, and it leaks forever. +-- Plain FOR UPDATE instead WAITS for the lock and, once it releases, re-checks the +-- row against the WHERE (Postgres EvalPlanQual): still dispatched/running → +-- included on this page; already failed by whoever held the lock → correctly +-- excluded. The bounded page size keeps that wait short, and recovery only races +-- short sweeper/reclaim transactions (both SKIP LOCKED, so they never wait — no +-- deadlock cycle is possible). -- -- Keyset cursor (MUL-4332 review round 3, point 1): the registration path upserts -- the runtime back to `online`, so the every-tick offline sweep will NOT reap an @@ -730,7 +744,7 @@ WHERE runtime_id = @runtime_id AND status IN ('dispatched', 'running', 'waiting_ ) ORDER BY created_at ASC, id ASC LIMIT @max_per_tick::int -FOR UPDATE SKIP LOCKED; +FOR UPDATE; -- name: SelectStaleTasksToFail :many -- Selects (and row-locks) up to @max_per_tick tasks stuck in dispatched/running From 0f0a6d6084966e779613b8a316e866e0ce66da36 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 16:47:46 +0800 Subject: [PATCH 06/15] feat(automation): add hooks persistence model Co-authored-by: multica-agent --- server/migrations/208_hook.down.sql | 1 + server/migrations/208_hook.up.sql | 24 ++++++++++++++++++ server/migrations/209_hook_revision.down.sql | 1 + server/migrations/209_hook_revision.up.sql | 16 ++++++++++++ .../migrations/210_automation_state.down.sql | 1 + server/migrations/210_automation_state.up.sql | 12 +++++++++ server/migrations/211_hook_execution.down.sql | 1 + server/migrations/211_hook_execution.up.sql | 25 +++++++++++++++++++ .../212_hook_action_effect.down.sql | 1 + .../migrations/212_hook_action_effect.up.sql | 19 ++++++++++++++ .../213_hook_revision_unique_index.down.sql | 1 + .../213_hook_revision_unique_index.up.sql | 1 + .../214_hook_execution_unique_index.down.sql | 1 + .../214_hook_execution_unique_index.up.sql | 1 + .../215_hook_effect_key_unique_index.down.sql | 1 + .../215_hook_effect_key_unique_index.up.sql | 1 + .../216_hook_execution_lease_index.down.sql | 1 + .../216_hook_execution_lease_index.up.sql | 1 + .../migrations/217_hook_scope_index.down.sql | 1 + server/migrations/217_hook_scope_index.up.sql | 1 + 20 files changed, 111 insertions(+) create mode 100644 server/migrations/208_hook.down.sql create mode 100644 server/migrations/208_hook.up.sql create mode 100644 server/migrations/209_hook_revision.down.sql create mode 100644 server/migrations/209_hook_revision.up.sql create mode 100644 server/migrations/210_automation_state.down.sql create mode 100644 server/migrations/210_automation_state.up.sql create mode 100644 server/migrations/211_hook_execution.down.sql create mode 100644 server/migrations/211_hook_execution.up.sql create mode 100644 server/migrations/212_hook_action_effect.down.sql create mode 100644 server/migrations/212_hook_action_effect.up.sql create mode 100644 server/migrations/213_hook_revision_unique_index.down.sql create mode 100644 server/migrations/213_hook_revision_unique_index.up.sql create mode 100644 server/migrations/214_hook_execution_unique_index.down.sql create mode 100644 server/migrations/214_hook_execution_unique_index.up.sql create mode 100644 server/migrations/215_hook_effect_key_unique_index.down.sql create mode 100644 server/migrations/215_hook_effect_key_unique_index.up.sql create mode 100644 server/migrations/216_hook_execution_lease_index.down.sql create mode 100644 server/migrations/216_hook_execution_lease_index.up.sql create mode 100644 server/migrations/217_hook_scope_index.down.sql create mode 100644 server/migrations/217_hook_scope_index.up.sql diff --git a/server/migrations/208_hook.down.sql b/server/migrations/208_hook.down.sql new file mode 100644 index 00000000000..d89376e88b4 --- /dev/null +++ b/server/migrations/208_hook.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook; diff --git a/server/migrations/208_hook.up.sql b/server/migrations/208_hook.up.sql new file mode 100644 index 00000000000..685a669af10 --- /dev/null +++ b/server/migrations/208_hook.up.sql @@ -0,0 +1,24 @@ +-- Event Hooks MVP — stable hook identity and lifecycle ownership. +-- UUID associations are application-validated; do not add FKs or cascades. +CREATE TABLE hook ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL, + name TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + active_revision_id UUID NOT NULL, + scope_type TEXT NOT NULL CHECK (scope_type IN ('workspace', 'issue')), + scope_id UUID, + retire_after_event_seq BIGINT, + origin TEXT NOT NULL DEFAULT 'user' CHECK (origin IN ('user', 'system')), + system_key TEXT, + system_version INT, + creator_actor_type TEXT NOT NULL, + creator_actor_id UUID, + authorization_principal_user_id UUID, + disabled_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + archived_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook IS 'Event Hooks stable identity and lifecycle owner. No foreign keys; application validates all associations.'; diff --git a/server/migrations/209_hook_revision.down.sql b/server/migrations/209_hook_revision.down.sql new file mode 100644 index 00000000000..d023801b896 --- /dev/null +++ b/server/migrations/209_hook_revision.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_revision; diff --git a/server/migrations/209_hook_revision.up.sql b/server/migrations/209_hook_revision.up.sql new file mode 100644 index 00000000000..3123dec7bb6 --- /dev/null +++ b/server/migrations/209_hook_revision.up.sql @@ -0,0 +1,16 @@ +-- Immutable configurations. Updating a hook creates a new revision row. +CREATE TABLE hook_revision ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + hook_id UUID NOT NULL, + revision INT NOT NULL, + event_type TEXT NOT NULL, + match JSONB NOT NULL DEFAULT '{}'::jsonb, + conditions JSONB, + fire_mode TEXT NOT NULL CHECK (fire_mode IN ('per_event', 'rising_edge')), + actions JSONB NOT NULL, + created_by_type TEXT NOT NULL, + created_by_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +COMMENT ON TABLE hook_revision IS 'Immutable Event Hooks configuration revisions. No foreign keys; application validates hook ownership.'; diff --git a/server/migrations/210_automation_state.down.sql b/server/migrations/210_automation_state.down.sql new file mode 100644 index 00000000000..85174bd27eb --- /dev/null +++ b/server/migrations/210_automation_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS automation_state; diff --git a/server/migrations/210_automation_state.up.sql b/server/migrations/210_automation_state.up.sql new file mode 100644 index 00000000000..e688b127285 --- /dev/null +++ b/server/migrations/210_automation_state.up.sql @@ -0,0 +1,12 @@ +-- Small durable, row-lockable state for edge latches, stage frontier and rate buckets. +CREATE TABLE automation_state ( + workspace_id UUID NOT NULL, + state_kind TEXT NOT NULL, + state_key TEXT NOT NULL, + state JSONB NOT NULL, + version BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, state_kind, state_key) +); + +COMMENT ON TABLE automation_state IS 'Durable row-lockable automation state, not an audit log. No foreign keys.'; diff --git a/server/migrations/211_hook_execution.down.sql b/server/migrations/211_hook_execution.down.sql new file mode 100644 index 00000000000..9ca147a0300 --- /dev/null +++ b/server/migrations/211_hook_execution.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_execution; diff --git a/server/migrations/211_hook_execution.up.sql b/server/migrations/211_hook_execution.up.sql new file mode 100644 index 00000000000..b2716853bb7 --- /dev/null +++ b/server/migrations/211_hook_execution.up.sql @@ -0,0 +1,25 @@ +-- Matcher decision and executor trace. The revision is pinned at creation. +CREATE TABLE hook_execution ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL, + hook_id UUID NOT NULL, + hook_revision_id UUID NOT NULL, + event_id UUID NOT NULL, + correlation_id UUID NOT NULL, + status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed', 'skipped')), + skip_reason TEXT, + match_snapshot JSONB, + condition_snapshot JSONB, + current_action_index INT NOT NULL DEFAULT 0, + attempts INT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ, + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + error_code TEXT, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook_execution IS 'Durable matcher/executor trace with revision pinning. No foreign keys; application validates all associations.'; diff --git a/server/migrations/212_hook_action_effect.down.sql b/server/migrations/212_hook_action_effect.down.sql new file mode 100644 index 00000000000..73ceedbcb09 --- /dev/null +++ b/server/migrations/212_hook_action_effect.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS hook_action_effect; diff --git a/server/migrations/212_hook_action_effect.up.sql b/server/migrations/212_hook_action_effect.up.sql new file mode 100644 index 00000000000..2d3a06ed0ee --- /dev/null +++ b/server/migrations/212_hook_action_effect.up.sql @@ -0,0 +1,19 @@ +-- Crash-safe idempotency anchor for one execution action. +CREATE TABLE hook_action_effect ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + effect_key TEXT NOT NULL, + execution_id UUID NOT NULL, + action_index INT NOT NULL, + action_type TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('running', 'succeeded', 'failed', 'skipped')), + resolved_input JSONB, + output_type TEXT, + output_id UUID, + attempts INT NOT NULL DEFAULT 0, + error_code TEXT, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ +); + +COMMENT ON TABLE hook_action_effect IS 'One durable idempotency record per hook execution action. No foreign keys.'; diff --git a/server/migrations/213_hook_revision_unique_index.down.sql b/server/migrations/213_hook_revision_unique_index.down.sql new file mode 100644 index 00000000000..147f1b57d6c --- /dev/null +++ b/server/migrations/213_hook_revision_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_revision_unique; diff --git a/server/migrations/213_hook_revision_unique_index.up.sql b/server/migrations/213_hook_revision_unique_index.up.sql new file mode 100644 index 00000000000..c33e02ff018 --- /dev/null +++ b/server/migrations/213_hook_revision_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_revision_unique ON hook_revision (hook_id, revision); diff --git a/server/migrations/214_hook_execution_unique_index.down.sql b/server/migrations/214_hook_execution_unique_index.down.sql new file mode 100644 index 00000000000..7319449246a --- /dev/null +++ b/server/migrations/214_hook_execution_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_execution_hook_event; diff --git a/server/migrations/214_hook_execution_unique_index.up.sql b/server/migrations/214_hook_execution_unique_index.up.sql new file mode 100644 index 00000000000..c9e62d13eb4 --- /dev/null +++ b/server/migrations/214_hook_execution_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_execution_hook_event ON hook_execution (hook_id, event_id); diff --git a/server/migrations/215_hook_effect_key_unique_index.down.sql b/server/migrations/215_hook_effect_key_unique_index.down.sql new file mode 100644 index 00000000000..1494263f79c --- /dev/null +++ b/server/migrations/215_hook_effect_key_unique_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_action_effect_key; diff --git a/server/migrations/215_hook_effect_key_unique_index.up.sql b/server/migrations/215_hook_effect_key_unique_index.up.sql new file mode 100644 index 00000000000..a9a07f65651 --- /dev/null +++ b/server/migrations/215_hook_effect_key_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX CONCURRENTLY idx_hook_action_effect_key ON hook_action_effect (effect_key); diff --git a/server/migrations/216_hook_execution_lease_index.down.sql b/server/migrations/216_hook_execution_lease_index.down.sql new file mode 100644 index 00000000000..a07995618e0 --- /dev/null +++ b/server/migrations/216_hook_execution_lease_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_execution_lease; diff --git a/server/migrations/216_hook_execution_lease_index.up.sql b/server/migrations/216_hook_execution_lease_index.up.sql new file mode 100644 index 00000000000..8025152b492 --- /dev/null +++ b/server/migrations/216_hook_execution_lease_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY idx_hook_execution_lease ON hook_execution (status, next_attempt_at, lease_expires_at, created_at); diff --git a/server/migrations/217_hook_scope_index.down.sql b/server/migrations/217_hook_scope_index.down.sql new file mode 100644 index 00000000000..fc48c705056 --- /dev/null +++ b/server/migrations/217_hook_scope_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY IF EXISTS idx_hook_scope_active; diff --git a/server/migrations/217_hook_scope_index.up.sql b/server/migrations/217_hook_scope_index.up.sql new file mode 100644 index 00000000000..a1fed9907d3 --- /dev/null +++ b/server/migrations/217_hook_scope_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY idx_hook_scope_active ON hook (workspace_id, scope_type, scope_id) WHERE archived_at IS NULL; From 500914052c31c4336e11fe4e684937fc964b0288 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 18:19:03 +0800 Subject: [PATCH 07/15] =?UTF-8?q?feat(automation):=20hook=20policy=20layer?= =?UTF-8?q?=20=E2=80=94=20validator=20+=20CRUD=20API/CLI=20(MUL-4332=20PR2?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the Event Hooks policy & debug layer on top of the PR2 persistence model (hook/revision/state/execution/effect tables, 208-217). Store-only and gated behind the automation_event_hooks feature flag (default off), so it adds zero runtime behavior until the matcher/executor slice ships: creating hooks persists rows that nothing yet consumes. - internal/automation: the user-authored hook spec (when/if/fire/do), the machine-readable event schema registry that declares which envelope/payload fields each domain event exposes for matching (§5.3/§6), the fixed condition and action vocabularies, and a pure typed Validator. The validator rejects an illegal rule at the API boundary (unknown/undeclared fields, bad uuids, wrong action shape, >8 actions, system-only actions on a user hook) and enforces the rising_edge dependency rule: a latch hook must listen to exactly the event that can change its condition (§5.2). - pkg/db/queries/hook.sql: workspace-scoped CRUD for hooks, immutable revisions, and the execution trace. Hook + revision #1 are created together with app-generated ids (they reference each other, no FK). - internal/service/hook.go: HookService — create (validate → resolve accountable principal → tx insert hook+revision), get, list, patch (append immutable revision + repoint active), enable/disable, soft-archive, executions read. System-managed hooks are not editable through this API (§8). - internal/handler/hook.go + routes: /api/hooks REST surface, every endpoint gated on the feature flag (404 when off) and resolving the audit creator + accountable human principal — an agent author with no resolvable member is refused (§8). - cmd/multica/cmd_hook.go: hook create/list/get/update/enable/disable/delete/ executions CLI. Tests: automation validator units (accepts A2 join / per_event, rejects 20+ illegal shapes, rising-edge coverage, ParseMatch clauses); handler CRUD lifecycle (create→get→update-appends-revision-2→list→disable→enable→ executions→soft-archive→404), feature-flag-off 404, invalid-spec 400, and agent-without-principal 403. Remaining MVP slices continue in this PR: PR3 matcher/executor + guardrails, PR4 actions/autopilot, PR5 stage sensor + system hooks. The execution flag stays off until all guardrails exist (§14). Co-authored-by: multica-agent --- server/cmd/multica/cmd_hook.go | 280 +++++++++++ server/cmd/multica/main.go | 2 + server/cmd/server/router.go | 16 + server/internal/automation/schema.go | 157 ++++++ server/internal/automation/spec.go | 193 ++++++++ server/internal/automation/validate.go | 278 +++++++++++ server/internal/automation/validate_test.go | 166 +++++++ server/internal/handler/handler.go | 2 + server/internal/handler/hook.go | 364 ++++++++++++++ server/internal/handler/hook_test.go | 240 ++++++++++ server/internal/service/hook.go | 344 ++++++++++++++ server/internal/util/pgx.go | 8 + server/pkg/db/generated/hook.sql.go | 502 ++++++++++++++++++++ server/pkg/db/generated/models.go | 89 ++++ server/pkg/db/queries/hook.sql | 89 ++++ 15 files changed, 2730 insertions(+) create mode 100644 server/cmd/multica/cmd_hook.go create mode 100644 server/internal/automation/schema.go create mode 100644 server/internal/automation/spec.go create mode 100644 server/internal/automation/validate.go create mode 100644 server/internal/automation/validate_test.go create mode 100644 server/internal/handler/hook.go create mode 100644 server/internal/handler/hook_test.go create mode 100644 server/internal/service/hook.go create mode 100644 server/pkg/db/generated/hook.sql.go create mode 100644 server/pkg/db/queries/hook.sql diff --git a/server/cmd/multica/cmd_hook.go b/server/cmd/multica/cmd_hook.go new file mode 100644 index 00000000000..135dc397c55 --- /dev/null +++ b/server/cmd/multica/cmd_hook.go @@ -0,0 +1,280 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/multica-ai/multica/server/internal/cli" +) + +var hookCmd = &cobra.Command{ + Use: "hook", + Short: "Work with event hooks (automation rules)", +} + +var hookListCmd = &cobra.Command{ + Use: "list", + Short: "List hooks in the workspace", + RunE: runHookList, +} + +var hookGetCmd = &cobra.Command{ + Use: "get ", + Short: "Get a hook and its active revision", + Args: exactArgs(1), + RunE: runHookGet, +} + +var hookCreateCmd = &cobra.Command{ + Use: "create", + Short: "Create a hook from a JSON spec file", + RunE: runHookCreate, +} + +var hookUpdateCmd = &cobra.Command{ + Use: "update ", + Short: "Update a hook from a JSON spec file (appends a new revision)", + Args: exactArgs(1), + RunE: runHookUpdate, +} + +var hookEnableCmd = &cobra.Command{ + Use: "enable ", + Short: "Enable a hook", + Args: exactArgs(1), + RunE: runHookEnable, +} + +var hookDisableCmd = &cobra.Command{ + Use: "disable ", + Short: "Disable a hook", + Args: exactArgs(1), + RunE: runHookDisable, +} + +var hookDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Archive (soft-delete) a hook", + Args: exactArgs(1), + RunE: runHookDelete, +} + +var hookExecutionsCmd = &cobra.Command{ + Use: "executions ", + Short: "List a hook's recent execution trace", + Args: exactArgs(1), + RunE: runHookExecutions, +} + +func init() { + hookCmd.AddCommand(hookListCmd) + hookCmd.AddCommand(hookGetCmd) + hookCmd.AddCommand(hookCreateCmd) + hookCmd.AddCommand(hookUpdateCmd) + hookCmd.AddCommand(hookEnableCmd) + hookCmd.AddCommand(hookDisableCmd) + hookCmd.AddCommand(hookDeleteCmd) + hookCmd.AddCommand(hookExecutionsCmd) + + hookListCmd.Flags().String("output", "table", "Output format: table or json") + hookGetCmd.Flags().String("output", "json", "Output format: table or json") + hookExecutionsCmd.Flags().String("output", "table", "Output format: table or json") + + hookCreateCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookCreateCmd.MarkFlagRequired("file") + hookUpdateCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookUpdateCmd.MarkFlagRequired("file") + + hookDisableCmd.Flags().String("reason", "", "Optional reason recorded on the hook") +} + +// readHookSpecFile loads and validates that the spec file is well-formed JSON, +// then returns it for the request body. The server performs full typed validation. +func readHookSpecFile(path string) (any, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read spec file: %w", err) + } + var body any + if err := json.Unmarshal(data, &body); err != nil { + return nil, fmt.Errorf("spec file is not valid JSON: %w", err) + } + return body, nil +} + +func runHookList(cmd *cobra.Command, _ []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var hooks []map[string]any + if err := client.GetJSON(ctx, "/api/hooks", &hooks); err != nil { + return fmt.Errorf("list hooks: %w", err) + } + output, _ := cmd.Flags().GetString("output") + if output == "json" { + return cli.PrintJSON(os.Stdout, hooks) + } + headers := []string{"ID", "NAME", "EVENT", "FIRE", "ENABLED"} + rows := make([][]string, 0, len(hooks)) + for _, hook := range hooks { + rows = append(rows, []string{ + strVal(hook, "id"), + strVal(hook, "name"), + hookRevisionField(hook, "event"), + hookRevisionField(hook, "fire_mode"), + fmt.Sprintf("%v", hook["enabled"]), + }) + } + cli.PrintTable(os.Stdout, headers, rows) + return nil +} + +func runHookGet(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var hook map[string]any + if err := client.GetJSON(ctx, "/api/hooks/"+args[0], &hook); err != nil { + return fmt.Errorf("get hook: %w", err) + } + return cli.PrintJSON(os.Stdout, hook) +} + +func runHookCreate(cmd *cobra.Command, _ []string) error { + path, _ := cmd.Flags().GetString("file") + body, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks", body, &result); err != nil { + return fmt.Errorf("create hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookUpdate(cmd *cobra.Command, args []string) error { + path, _ := cmd.Flags().GetString("file") + body, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PatchJSON(ctx, "/api/hooks/"+args[0], body, &result); err != nil { + return fmt.Errorf("update hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookEnable(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/"+args[0]+"/enable", map[string]any{}, &result); err != nil { + return fmt.Errorf("enable hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookDisable(cmd *cobra.Command, args []string) error { + reason, _ := cmd.Flags().GetString("reason") + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/"+args[0]+"/disable", map[string]any{"reason": reason}, &result); err != nil { + return fmt.Errorf("disable hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookDelete(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + if err := client.DeleteJSON(ctx, "/api/hooks/"+args[0]); err != nil { + return fmt.Errorf("delete hook: %w", err) + } + fmt.Printf("Hook archived: %s\n", args[0]) + return nil +} + +func runHookExecutions(cmd *cobra.Command, args []string) error { + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var execs []map[string]any + if err := client.GetJSON(ctx, "/api/hooks/"+args[0]+"/executions", &execs); err != nil { + return fmt.Errorf("list hook executions: %w", err) + } + output, _ := cmd.Flags().GetString("output") + if output == "json" { + return cli.PrintJSON(os.Stdout, execs) + } + headers := []string{"ID", "STATUS", "SKIP_REASON", "EVENT_ID", "CREATED_AT"} + rows := make([][]string, 0, len(execs)) + for _, e := range execs { + rows = append(rows, []string{ + strVal(e, "id"), + strVal(e, "status"), + strVal(e, "skip_reason"), + strVal(e, "event_id"), + strVal(e, "created_at"), + }) + } + cli.PrintTable(os.Stdout, headers, rows) + return nil +} + +// hookRevisionField reads a field out of the nested "revision" object for the +// list table view. +func hookRevisionField(hook map[string]any, key string) string { + rev, ok := hook["revision"].(map[string]any) + if !ok { + return "" + } + return strVal(rev, key) +} diff --git a/server/cmd/multica/main.go b/server/cmd/multica/main.go index 7491fd18da7..9d521b5b941 100644 --- a/server/cmd/multica/main.go +++ b/server/cmd/multica/main.go @@ -53,6 +53,7 @@ func init() { repoCmd.GroupID = groupCore skillCmd.GroupID = groupCore squadCmd.GroupID = groupCore + hookCmd.GroupID = groupCore chatCmd.GroupID = groupCore // Runtime commands @@ -79,6 +80,7 @@ func init() { rootCmd.AddCommand(repoCmd) rootCmd.AddCommand(skillCmd) rootCmd.AddCommand(squadCmd) + rootCmd.AddCommand(hookCmd) rootCmd.AddCommand(chatCmd) rootCmd.AddCommand(daemonCmd) rootCmd.AddCommand(runtimeCmd) diff --git a/server/cmd/server/router.go b/server/cmd/server/router.go index 9b1eb3e7d50..d8c3c209096 100644 --- a/server/cmd/server/router.go +++ b/server/cmd/server/router.go @@ -1248,6 +1248,22 @@ func NewRouterWithOptions(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus }) }) + // Event Hooks (MUL-4332) - automation rules CRUD. Every handler + // additionally gates on the automation_event_hooks feature flag + // (default off), so this surface is invisible until enabled. + r.Route("/api/hooks", func(r chi.Router) { + r.Get("/", h.ListHooks) + r.Post("/", h.CreateHook) + r.Route("/{id}", func(r chi.Router) { + r.Get("/", h.GetHook) + r.Patch("/", h.UpdateHook) + r.Delete("/", h.DeleteHook) + r.Post("/enable", h.EnableHook) + r.Post("/disable", h.DisableHook) + r.Get("/executions", h.ListHookExecutions) + }) + }) + // Dashboard — workspace-wide token + run-time rollups for the // "/{slug}/dashboard" page. Optional ?project_id filter scopes // the rollup to a single project. diff --git a/server/internal/automation/schema.go b/server/internal/automation/schema.go new file mode 100644 index 00000000000..2ad09857701 --- /dev/null +++ b/server/internal/automation/schema.go @@ -0,0 +1,157 @@ +package automation + +import "github.com/multica-ai/multica/server/internal/domainevent" + +// FieldKind classifies a matchable event field so the validator can range-check +// clause values (e.g. a uuid field rejects a non-uuid literal). +type FieldKind int + +const ( + FieldString FieldKind = iota + FieldUUID +) + +// EventSchema declares, for one domain event type, which envelope/payload field +// paths a hook may match on. The design (§5.3/§6) requires that "字段路径必须由 +// 对应 event schema 声明" — this registry is that declaration, kept in lockstep +// with the domainevent payload structs. Only user-triggerable events are listed; +// system-derived events (issue.stage_completed) are not user-authorable in v1. +type EventSchema struct { + Type string + MatchFields map[string]FieldKind +} + +// envelopeMatchFields are common to every event type (the domain_event envelope +// columns a hook may match on). subject_id is the primary join key; actor_* let +// a rule scope to who caused the event. +var envelopeMatchFields = map[string]FieldKind{ + "subject_id": FieldUUID, + "actor_type": FieldString, + "actor_id": FieldUUID, +} + +// eventSchemas is the authoritative match-field registry, one entry per +// user-authorable v1 event type. Payload fields mirror the domainevent payload +// structs' json tags exactly. +var eventSchemas = buildEventSchemas(map[string]map[string]FieldKind{ + domainevent.TypeIssueCreated: { + "status": FieldString, + "priority": FieldString, + "parent_issue_id": FieldUUID, + "assignee_type": FieldString, + "assignee_id": FieldUUID, + "origin_type": FieldString, + }, + domainevent.TypeIssueStatusChanged: { + "from": FieldString, + "to": FieldString, + }, + domainevent.TypeIssueAssigned: { + "from_assignee_type": FieldString, + "from_assignee_id": FieldUUID, + "to_assignee_type": FieldString, + "to_assignee_id": FieldUUID, + }, + domainevent.TypeCommentCreated: { + "issue_id": FieldUUID, + "author_type": FieldString, + "author_id": FieldUUID, + "parent_id": FieldUUID, + }, + domainevent.TypeTaskCompleted: { + "issue_id": FieldUUID, + "agent_id": FieldUUID, + }, + domainevent.TypeTaskFailed: { + "issue_id": FieldUUID, + "agent_id": FieldUUID, + "retry_eligible": FieldString, + "error_code": FieldString, + }, +}) + +func buildEventSchemas(payloadFields map[string]map[string]FieldKind) map[string]EventSchema { + out := make(map[string]EventSchema, len(payloadFields)) + for evtType, fields := range payloadFields { + merged := make(map[string]FieldKind, len(fields)+len(envelopeMatchFields)) + for k, v := range envelopeMatchFields { + merged[k] = v + } + for k, v := range fields { + merged[k] = v + } + out[evtType] = EventSchema{Type: evtType, MatchFields: merged} + } + return out +} + +// SchemaFor returns the match-field schema for a user-authorable event type. +func SchemaFor(eventType string) (EventSchema, bool) { + s, ok := eventSchemas[eventType] + return s, ok +} + +// Action types. User actions are creatable through the public API; system-only +// actions are reserved for managed system hooks (PR5) and are rejected on a +// user-authored spec. +const ( + ActionSetIssueStatus = "set_issue_status" + ActionTriggerAgent = "trigger_agent" + ActionAddComment = "add_comment" + ActionSendInbox = "send_inbox" + ActionRunAutopilot = "run_autopilot" + + ActionSetIssueStatusMany = "set_issue_status_many" // system-only + ActionTriggerIssueAssignee = "trigger_issue_assignee" // system-only +) + +// userActionTypes is the set of action types a user-authored hook may use. +var userActionTypes = map[string]bool{ + ActionSetIssueStatus: true, + ActionTriggerAgent: true, + ActionAddComment: true, + ActionSendInbox: true, + ActionRunAutopilot: true, +} + +// systemActionTypes is the set of action types reserved for managed system +// hooks; a user spec that names one is rejected (§8 — user hooks do not inherit +// the system routing bypass). +var systemActionTypes = map[string]bool{ + ActionSetIssueStatusMany: true, + ActionTriggerIssueAssignee: true, +} + +// issue field names valid in an issue_field condition (§5.4). +const ( + IssueFieldStatus = "status" + IssueFieldAssigneeID = "assignee_id" + IssueFieldParentIssueID = "parent_issue_id" +) + +var validIssueFields = map[string]bool{ + IssueFieldStatus: true, + IssueFieldAssigneeID: true, + IssueFieldParentIssueID: true, +} + +// conditionDependencyEvent maps a condition to the single v1 domain event that +// can change its truth value. A rising_edge hook must listen to exactly that +// event so its latch can be re-evaluated (§5.2). In the v1 fixed vocabulary +// each condition depends on exactly one event type. +func conditionDependencyEvent(c ConditionSpec) (string, bool) { + switch { + case c.IssuesStatus != nil: + return domainevent.TypeIssueStatusChanged, true + case c.IssueField != nil: + switch c.IssueField.Field { + case IssueFieldStatus: + return domainevent.TypeIssueStatusChanged, true + case IssueFieldAssigneeID: + return domainevent.TypeIssueAssigned, true + case IssueFieldParentIssueID: + return domainevent.TypeIssueCreated, true + } + } + return "", false +} diff --git a/server/internal/automation/spec.go b/server/internal/automation/spec.go new file mode 100644 index 00000000000..8064f0657f2 --- /dev/null +++ b/server/internal/automation/spec.go @@ -0,0 +1,193 @@ +// Package automation implements the Event Hooks MVP policy layer (issue +// MUL-4332): the user-authored hook specification, the versioned event schema +// registry every rule binds against, and the typed validator that rejects an +// illegal rule at the API boundary rather than deep inside a worker. +// +// This package is pure policy/config: it parses and validates hook specs and +// declares the fixed vocabulary of events, conditions and actions. It performs +// NO matching and NO execution — the durable matcher/executor is a later slice +// (PR3+) and will reuse the parsed spec and the schema registry defined here. +package automation + +import ( + "encoding/json" + "fmt" +) + +// Fire modes (hook_revision.fire_mode). +const ( + FirePerEvent = "per_event" + FireRisingEdge = "rising_edge" +) + +// Scope types (hook.scope_type). An issue-scoped hook is owned by / displayed +// on an issue; it does not implicitly restrict the event subject — that is the +// job of `when`. +const ( + ScopeWorkspace = "workspace" + ScopeIssue = "issue" +) + +// HookSpec is the user-authored hook definition — the POST/PATCH request body. +// It maps onto two rows: the hook (name, scope) and its immutable revision +// (event_type, match, conditions, fire_mode, actions). +type HookSpec struct { + Name string `json:"name"` + Scope *ScopeSpec `json:"scope,omitempty"` + When WhenSpec `json:"when"` + If []ConditionSpec `json:"if,omitempty"` + Fire FireSpec `json:"fire"` + Do []ActionSpec `json:"do"` +} + +// ScopeSpec is the hook lifecycle owner. Absent means workspace scope. +type ScopeSpec struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` +} + +// WhenSpec selects the triggering event and an optional set of match clauses on +// its envelope/payload fields. Match is kept raw here and parsed/validated by +// ParseMatch against the event's schema so the wire form and the typed model +// stay decoupled. +type WhenSpec struct { + Event string `json:"event"` + Match json.RawMessage `json:"match,omitempty"` +} + +// FireSpec selects per_event or rising_edge semantics. +type FireSpec struct { + Mode string `json:"mode"` +} + +// ConditionSpec is one predicate over current workspace state. Exactly one of +// the fixed-vocabulary variants must be set (§5.4). +type ConditionSpec struct { + IssuesStatus *IssuesStatusCond `json:"issues_status,omitempty"` + IssueField *IssueFieldCond `json:"issue_field,omitempty"` +} + +// IssuesStatusCond asserts every (all) or any of the given issues is in a +// status. Exactly one of All/Any is set. +type IssuesStatusCond struct { + IDs []string `json:"ids"` + All string `json:"all,omitempty"` + Any string `json:"any,omitempty"` +} + +// IssueFieldCond asserts a single issue field equals / is in a set of values. +type IssueFieldCond struct { + ID string `json:"id"` + Field string `json:"field"` + Eq string `json:"eq,omitempty"` + In []string `json:"in,omitempty"` +} + +// ActionSpec is one action in the ordered `do` list. Fields are a superset over +// all action types; the validator enforces the required/allowed set per type. +// Parameters are literals in v1 (event-field binding and templating land with +// the executor slice). +type ActionSpec struct { + Type string `json:"type"` + IssueID string `json:"issue_id,omitempty"` + Status string `json:"status,omitempty"` + AgentID string `json:"agent_id,omitempty"` + MemberID string `json:"member_id,omitempty"` + Message string `json:"message,omitempty"` + AutopilotID string `json:"autopilot_id,omitempty"` +} + +// MatchOp is the operator of a single match clause. +type MatchOp string + +const ( + MatchEq MatchOp = "eq" + MatchIn MatchOp = "in" + MatchExists MatchOp = "exists" +) + +// MatchClause is one parsed match predicate on a single event field path. +type MatchClause struct { + Op MatchOp + Value string // for eq + Set []string // for in + Exists bool // for exists +} + +// Match is the parsed set of clauses keyed by event field path. +type Match map[string]MatchClause + +// ParseMatch decodes the raw `when.match` object into typed clauses. A value is +// either a JSON scalar (→ eq), an object {"in": [...]} (→ in) or an object +// {"exists": true|false} (→ exists). It does NOT check field paths against a +// schema — that is Validator.validateMatch's job — so it can be reused by the +// executor once the field set is already known-valid. +func ParseMatch(raw json.RawMessage) (Match, error) { + if len(raw) == 0 { + return Match{}, nil + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, fmt.Errorf("match must be an object: %w", err) + } + out := make(Match, len(obj)) + for field, rawVal := range obj { + clause, err := parseMatchClause(rawVal) + if err != nil { + return nil, fmt.Errorf("match.%s: %w", field, err) + } + out[field] = clause + } + return out, nil +} + +func parseMatchClause(raw json.RawMessage) (MatchClause, error) { + // Try an object form first: {"in": [...]} or {"exists": bool}. + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err == nil { + switch { + case len(obj) != 1: + return MatchClause{}, fmt.Errorf("clause object must have exactly one of \"in\" or \"exists\"") + case obj["in"] != nil: + var set []string + if err := json.Unmarshal(obj["in"], &set); err != nil { + return MatchClause{}, fmt.Errorf("\"in\" must be a string array: %w", err) + } + if len(set) == 0 { + return MatchClause{}, fmt.Errorf("\"in\" must not be empty") + } + return MatchClause{Op: MatchIn, Set: set}, nil + case obj["exists"] != nil: + var exists bool + if err := json.Unmarshal(obj["exists"], &exists); err != nil { + return MatchClause{}, fmt.Errorf("\"exists\" must be a boolean: %w", err) + } + return MatchClause{Op: MatchExists, Exists: exists}, nil + default: + return MatchClause{}, fmt.Errorf("unsupported clause; use a scalar, {\"in\": [...]} or {\"exists\": bool}") + } + } + // Otherwise a scalar equality. Accept string/number/bool, normalised to string. + var scalar any + if err := json.Unmarshal(raw, &scalar); err != nil { + return MatchClause{}, fmt.Errorf("value must be a scalar or clause object: %w", err) + } + switch v := scalar.(type) { + case string: + return MatchClause{Op: MatchEq, Value: v}, nil + case bool: + return MatchClause{Op: MatchEq, Value: fmt.Sprintf("%t", v)}, nil + case float64: + return MatchClause{Op: MatchEq, Value: formatNumber(v)}, nil + default: + return MatchClause{}, fmt.Errorf("equality value must be a string, number or boolean") + } +} + +func formatNumber(f float64) string { + // Integers render without a trailing .0 so a match on an int field reads naturally. + if f == float64(int64(f)) { + return fmt.Sprintf("%d", int64(f)) + } + return fmt.Sprintf("%g", f) +} diff --git a/server/internal/automation/validate.go b/server/internal/automation/validate.go new file mode 100644 index 00000000000..ff712e3c329 --- /dev/null +++ b/server/internal/automation/validate.go @@ -0,0 +1,278 @@ +package automation + +import ( + "errors" + "fmt" + + "github.com/multica-ai/multica/server/internal/util" +) + +// Guardrail limits (§9). Rate/concurrency/depth limits belong to the executor +// slice; these are the shape limits enforced at author time. +const ( + MaxActionsPerHook = 8 + MaxNameLength = 200 +) + +// ValidationError is a user-fixable problem with a hook spec. Handlers map it to +// HTTP 400 (never 500) so a bad rule can never reach a worker (§13). +type ValidationError struct{ msg string } + +func (e *ValidationError) Error() string { return e.msg } + +func verr(format string, args ...any) error { + return &ValidationError{msg: fmt.Sprintf(format, args...)} +} + +// AsValidationError reports whether err is a ValidationError. +func AsValidationError(err error) (*ValidationError, bool) { + var ve *ValidationError + if errors.As(err, &ve) { + return ve, true + } + return nil, false +} + +// Validate performs complete typed validation of a user-authored hook spec: +// event schema, match fields, condition dependency, action schema, fire-mode +// coverage and shape limits. It is the single author-time gate reused by both +// the create and patch paths. +func Validate(spec HookSpec) error { + if spec.Name == "" { + return verr("name is required") + } + if len(spec.Name) > MaxNameLength { + return verr("name must be at most %d characters", MaxNameLength) + } + if err := validateScope(spec.Scope); err != nil { + return err + } + + schema, ok := SchemaFor(spec.When.Event) + if !ok { + return verr("unknown or non-authorable event type %q", spec.When.Event) + } + if err := validateMatch(spec.When.Match, schema); err != nil { + return err + } + + for i, cond := range spec.If { + if err := validateCondition(cond); err != nil { + return verr("if[%d]: %s", i, err.Error()) + } + } + + if err := validateFire(spec); err != nil { + return err + } + + if len(spec.Do) == 0 { + return verr("do must contain at least one action") + } + if len(spec.Do) > MaxActionsPerHook { + return verr("do must contain at most %d actions", MaxActionsPerHook) + } + for i, action := range spec.Do { + if err := validateAction(action); err != nil { + return verr("do[%d]: %s", i, err.Error()) + } + } + return nil +} + +func validateScope(scope *ScopeSpec) error { + if scope == nil { + return nil + } + switch scope.Type { + case ScopeWorkspace: + if scope.ID != "" { + return verr("scope.id must be empty for a workspace-scoped hook") + } + case ScopeIssue: + if !validUUID(scope.ID) { + return verr("scope.id must be a valid issue id for an issue-scoped hook") + } + default: + return verr("scope.type must be %q or %q", ScopeWorkspace, ScopeIssue) + } + return nil +} + +func validateMatch(raw []byte, schema EventSchema) error { + match, err := ParseMatch(raw) + if err != nil { + return verr("%s", err.Error()) + } + for field, clause := range match { + kind, ok := schema.MatchFields[field] + if !ok { + return verr("match field %q is not declared by event %q", field, schema.Type) + } + if err := validateClauseValues(field, kind, clause); err != nil { + return err + } + } + return nil +} + +func validateClauseValues(field string, kind FieldKind, clause MatchClause) error { + if kind != FieldUUID { + return nil // string fields accept any scalar; existence needs no value check + } + switch clause.Op { + case MatchEq: + if !validUUID(clause.Value) { + return verr("match field %q expects a uuid, got %q", field, clause.Value) + } + case MatchIn: + for _, v := range clause.Set { + if !validUUID(v) { + return verr("match field %q expects uuids, got %q", field, v) + } + } + } + return nil +} + +func validateCondition(c ConditionSpec) error { + set := 0 + if c.IssuesStatus != nil { + set++ + } + if c.IssueField != nil { + set++ + } + if set != 1 { + return verr("exactly one of issues_status or issue_field must be set") + } + if c.IssuesStatus != nil { + return validateIssuesStatus(*c.IssuesStatus) + } + return validateIssueField(*c.IssueField) +} + +func validateIssuesStatus(c IssuesStatusCond) error { + if len(c.IDs) == 0 { + return verr("issues_status.ids must not be empty") + } + for _, id := range c.IDs { + if !validUUID(id) { + return verr("issues_status.ids must be uuids, got %q", id) + } + } + hasAll, hasAny := c.All != "", c.Any != "" + if hasAll == hasAny { + return verr("exactly one of issues_status.all or issues_status.any must be set") + } + return nil +} + +func validateIssueField(c IssueFieldCond) error { + if !validUUID(c.ID) { + return verr("issue_field.id must be a uuid") + } + if !validIssueFields[c.Field] { + return verr("issue_field.field must be one of status, assignee_id, parent_issue_id") + } + hasEq, hasIn := c.Eq != "", len(c.In) > 0 + if hasEq == hasIn { + return verr("exactly one of issue_field.eq or issue_field.in must be set") + } + // Field-typed values: status is a free string; the id-shaped fields require uuids. + if c.Field != IssueFieldStatus { + if hasEq && !validUUID(c.Eq) { + return verr("issue_field.eq must be a uuid for field %q", c.Field) + } + for _, v := range c.In { + if !validUUID(v) { + return verr("issue_field.in must be uuids for field %q", c.Field) + } + } + } + return nil +} + +func validateFire(spec HookSpec) error { + switch spec.Fire.Mode { + case FirePerEvent: + return nil + case FireRisingEdge: + return validateRisingEdgeCoverage(spec) + default: + return verr("fire.mode must be %q or %q", FirePerEvent, FireRisingEdge) + } +} + +// validateRisingEdgeCoverage enforces §5.2: a rising_edge hook's latch can only +// be re-evaluated by the event it listens to, so every condition must depend on +// exactly the hook's own event type, and there must be at least one condition to +// gate on. This is the extractable dependency check the design requires at save +// time for the v1 fixed vocabulary. +func validateRisingEdgeCoverage(spec HookSpec) error { + if len(spec.If) == 0 { + return verr("rising_edge requires at least one condition in if") + } + for i, cond := range spec.If { + dep, ok := conditionDependencyEvent(cond) + if !ok { + return verr("if[%d]: condition has no known change event, cannot use rising_edge", i) + } + if dep != spec.When.Event { + return verr("rising_edge hook must listen to %q so its condition in if[%d] can be re-evaluated, but when.event is %q", dep, i, spec.When.Event) + } + } + return nil +} + +func validateAction(a ActionSpec) error { + if systemActionTypes[a.Type] { + return verr("action type %q is reserved for system hooks", a.Type) + } + if !userActionTypes[a.Type] { + return verr("unknown action type %q", a.Type) + } + switch a.Type { + case ActionSetIssueStatus: + if !validUUID(a.IssueID) { + return verr("set_issue_status requires a valid issue_id") + } + if a.Status == "" { + return verr("set_issue_status requires status") + } + case ActionTriggerAgent: + if !validUUID(a.IssueID) { + return verr("trigger_agent requires a valid issue_id") + } + if !validUUID(a.AgentID) { + return verr("trigger_agent requires a valid agent_id") + } + case ActionAddComment: + if !validUUID(a.IssueID) { + return verr("add_comment requires a valid issue_id") + } + if a.Message == "" { + return verr("add_comment requires message") + } + case ActionSendInbox: + if !validUUID(a.MemberID) { + return verr("send_inbox requires a valid member_id") + } + if a.Message == "" { + return verr("send_inbox requires message") + } + case ActionRunAutopilot: + if !validUUID(a.AutopilotID) { + return verr("run_autopilot requires a valid autopilot_id") + } + } + return nil +} + +func validUUID(s string) bool { + if s == "" { + return false + } + _, err := util.ParseUUID(s) + return err == nil +} diff --git a/server/internal/automation/validate_test.go b/server/internal/automation/validate_test.go new file mode 100644 index 00000000000..ce6d3839b5f --- /dev/null +++ b/server/internal/automation/validate_test.go @@ -0,0 +1,166 @@ +package automation + +import ( + "encoding/json" + "strings" + "testing" +) + +const ( + uuidA = "11111111-1111-1111-1111-111111111111" + uuidB = "22222222-2222-2222-2222-222222222222" + uuidC = "33333333-3333-3333-3333-333333333333" + uuidM = "44444444-4444-4444-4444-444444444444" +) + +// validPerEvent is the minimal accepted spec: comment.created → add a comment. +func validPerEvent() HookSpec { + return HookSpec{ + Name: "notify on comment", + When: WhenSpec{Event: "comment.created"}, + Fire: FireSpec{Mode: FirePerEvent}, + Do: []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "hi"}}, + } +} + +// validRisingEdge is the design's A2 join: A and B both done → start C + inbox. +func validRisingEdge() HookSpec { + return HookSpec{ + Name: "A/B done then start C", + When: WhenSpec{Event: "issue.status_changed", Match: json.RawMessage(`{"subject_id":{"in":["` + uuidA + `","` + uuidB + `"]}}`)}, + If: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + Fire: FireSpec{Mode: FireRisingEdge}, + Do: []ActionSpec{ + {Type: ActionSetIssueStatus, IssueID: uuidC, Status: "todo"}, + {Type: ActionSendInbox, MemberID: uuidM, Message: "A/B done, C started"}, + }, + } +} + +func TestValidateAcceptsValidSpecs(t *testing.T) { + for name, spec := range map[string]HookSpec{ + "per_event minimal": validPerEvent(), + "rising_edge join": validRisingEdge(), + } { + if err := Validate(spec); err != nil { + t.Errorf("%s: expected valid, got %v", name, err) + } + } +} + +func TestValidateRejectsInvalidSpecs(t *testing.T) { + cases := []struct { + name string + mutate func(*HookSpec) + wantSub string // substring the error message must contain + }{ + {"missing name", func(s *HookSpec) { s.Name = "" }, "name is required"}, + {"unknown event", func(s *HookSpec) { s.When.Event = "issue.exploded" }, "non-authorable event"}, + {"system event not authorable", func(s *HookSpec) { s.When.Event = "issue.stage_completed" }, "non-authorable event"}, + {"undeclared match field", func(s *HookSpec) { + s.When = WhenSpec{Event: "comment.created", Match: json.RawMessage(`{"nope":"x"}`)} + }, "not declared"}, + {"uuid match field non-uuid", func(s *HookSpec) { + s.When = WhenSpec{Event: "comment.created", Match: json.RawMessage(`{"issue_id":"not-a-uuid"}`)} + }, "expects a uuid"}, + {"empty do", func(s *HookSpec) { s.Do = nil }, "at least one action"}, + {"too many actions", func(s *HookSpec) { + s.Do = make([]ActionSpec, MaxActionsPerHook+1) + for i := range s.Do { + s.Do[i] = ActionSpec{Type: ActionAddComment, IssueID: uuidC, Message: "x"} + } + }, "at most"}, + {"system-only action", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatusMany}} + }, "reserved for system"}, + {"unknown action", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: "delete_workspace"}} + }, "unknown action type"}, + {"set_issue_status missing status", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatus, IssueID: uuidC}} + }, "requires status"}, + {"trigger_agent bad agent", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionTriggerAgent, IssueID: uuidC, AgentID: "x"}} + }, "valid agent_id"}, + {"bad fire mode", func(s *HookSpec) { s.Fire.Mode = "always" }, "fire.mode"}, + {"scope issue without id", func(s *HookSpec) { + s.Scope = &ScopeSpec{Type: ScopeIssue} + }, "issue-scoped"}, + {"scope workspace with id", func(s *HookSpec) { + s.Scope = &ScopeSpec{Type: ScopeWorkspace, ID: uuidA} + }, "must be empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + spec := validPerEvent() + tc.mutate(&spec) + err := Validate(spec) + if err == nil { + t.Fatalf("expected rejection, got nil") + } + if _, ok := AsValidationError(err); !ok { + t.Fatalf("expected *ValidationError, got %T", err) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub) + } + }) + } +} + +// The rising-edge dependency check (§5.2) is subtle enough to test on its own. +func TestValidateRisingEdgeCoverage(t *testing.T) { + t.Run("requires a condition", func(t *testing.T) { + spec := validRisingEdge() + spec.If = nil + if err := Validate(spec); err == nil || !strings.Contains(err.Error(), "at least one condition") { + t.Fatalf("rising_edge without conditions must be rejected, got %v", err) + } + }) + t.Run("must listen to the condition's change event", func(t *testing.T) { + spec := validRisingEdge() + // issues_status depends on issue.status_changed; listening to comment.created + // means the latch could never be re-evaluated. + spec.When = WhenSpec{Event: "comment.created"} + err := Validate(spec) + if err == nil || !strings.Contains(err.Error(), "rising_edge hook must listen to") { + t.Fatalf("rising_edge listening to the wrong event must be rejected, got %v", err) + } + }) + t.Run("assignee condition binds to issue.assigned", func(t *testing.T) { + spec := HookSpec{ + Name: "assignee gate", + When: WhenSpec{Event: "issue.assigned"}, + If: []ConditionSpec{{IssueField: &IssueFieldCond{ID: uuidC, Field: IssueFieldAssigneeID, Eq: uuidM}}}, + Fire: FireSpec{Mode: FireRisingEdge}, + Do: []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "assigned"}}, + } + if err := Validate(spec); err != nil { + t.Fatalf("assignee rising_edge on issue.assigned should be valid, got %v", err) + } + }) +} + +// ParseMatch is the shared wire→typed step; verify each clause shape. +func TestParseMatch(t *testing.T) { + m, err := ParseMatch(json.RawMessage(`{"to":"done","subject_id":{"in":["` + uuidA + `"]},"actor_id":{"exists":true}}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := m["to"]; got.Op != MatchEq || got.Value != "done" { + t.Errorf("to clause = %+v, want eq done", got) + } + if got := m["subject_id"]; got.Op != MatchIn || len(got.Set) != 1 || got.Set[0] != uuidA { + t.Errorf("subject_id clause = %+v, want in [%s]", got, uuidA) + } + if got := m["actor_id"]; got.Op != MatchExists || !got.Exists { + t.Errorf("actor_id clause = %+v, want exists true", got) + } + + if _, err := ParseMatch(json.RawMessage(`{"to":{"in":[]}}`)); err == nil { + t.Errorf("empty in set must be rejected") + } + if _, err := ParseMatch(json.RawMessage(`"notanobject"`)); err == nil { + t.Errorf("non-object match must be rejected") + } +} diff --git a/server/internal/handler/handler.go b/server/internal/handler/handler.go index 33543a12eaa..612c667ba8c 100644 --- a/server/internal/handler/handler.go +++ b/server/internal/handler/handler.go @@ -139,6 +139,7 @@ type Handler struct { TaskService *service.TaskService IssueService *service.IssueService AutopilotService *service.AutopilotService + HookService *service.HookService EmailService *service.EmailService UpdateStore UpdateStore ModelListStore ModelListStore @@ -275,6 +276,7 @@ func New(queries *db.Queries, txStarter txStarter, hub *realtime.Hub, bus *event TaskService: taskSvc, IssueService: service.NewIssueService(queries, txStarter, bus, analyticsClient, taskSvc), AutopilotService: service.NewAutopilotService(queries, txStarter, bus, taskSvc), + HookService: service.NewHookService(queries, txStarter), EmailService: emailService, UpdateStore: NewInMemoryUpdateStore(), ModelListStore: NewInMemoryModelListStore(), diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go new file mode 100644 index 00000000000..51a9ed2ddc0 --- /dev/null +++ b/server/internal/handler/hook.go @@ -0,0 +1,364 @@ +package handler + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// hookExecutionListLimit bounds the execution-trace read for the debug endpoint. +const hookExecutionListLimit = 100 + +// HookResponse is the API view of a hook plus its active revision. +type HookResponse struct { + ID string `json:"id"` + WorkspaceID string `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Scope HookScopeResponse `json:"scope"` + Origin string `json:"origin"` + DisabledReason string `json:"disabled_reason,omitempty"` + Revision HookRevisionResponse `json:"revision"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// HookScopeResponse is the hook lifecycle scope. +type HookScopeResponse struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` +} + +// HookRevisionResponse is the active immutable revision. +type HookRevisionResponse struct { + ID string `json:"id"` + Revision int32 `json:"revision"` + Event string `json:"event"` + Match json.RawMessage `json:"match"` + Conditions json.RawMessage `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions json.RawMessage `json:"actions"` + CreatedAt string `json:"created_at"` +} + +// HookExecutionResponse is one row of the execution trace (all null in PR2 — no +// matcher runs yet — but shaped so the debug endpoint is stable for PR3). +type HookExecutionResponse struct { + ID string `json:"id"` + HookID string `json:"hook_id"` + RevisionID string `json:"hook_revision_id"` + EventID string `json:"event_id"` + Correlation string `json:"correlation_id"` + Status string `json:"status"` + SkipReason string `json:"skip_reason,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + CreatedAt string `json:"created_at"` +} + +func hookToResponse(hr service.HookWithRevision) HookResponse { + h, rev := hr.Hook, hr.Revision + return HookResponse{ + ID: uuidToString(h.ID), + WorkspaceID: uuidToString(h.WorkspaceID), + Name: h.Name, + Enabled: h.Enabled, + Scope: HookScopeResponse{Type: h.ScopeType, ID: uuidToString(h.ScopeID)}, + Origin: h.Origin, + DisabledReason: textValue(h.DisabledReason), + Revision: HookRevisionResponse{ + ID: uuidToString(rev.ID), + Revision: rev.Revision, + Event: rev.EventType, + Match: rawJSON(rev.Match), + Conditions: rawJSON(rev.Conditions), + FireMode: rev.FireMode, + Actions: rawJSON(rev.Actions), + CreatedAt: timestampToString(rev.CreatedAt), + }, + CreatedAt: timestampToString(h.CreatedAt), + UpdatedAt: timestampToString(h.UpdatedAt), + } +} + +func hookExecutionToResponse(e db.HookExecution) HookExecutionResponse { + return HookExecutionResponse{ + ID: uuidToString(e.ID), + HookID: uuidToString(e.HookID), + RevisionID: uuidToString(e.HookRevisionID), + EventID: uuidToString(e.EventID), + Correlation: uuidToString(e.CorrelationID), + Status: e.Status, + SkipReason: textValue(e.SkipReason), + ErrorCode: textValue(e.ErrorCode), + CreatedAt: timestampToString(e.CreatedAt), + } +} + +// hookEnabled gates every hook endpoint behind the automation_event_hooks flag. +// Store-only PR2 already writes rows, but the whole surface stays invisible until +// the flag is deliberately turned on, so nothing changes for existing workspaces. +func (h *Handler) hookEnabled(w http.ResponseWriter, r *http.Request) bool { + if !featureflags.EventHooksEnabled(r.Context(), h.FeatureFlags) { + writeError(w, http.StatusNotFound, "event hooks are not enabled") + return false + } + return true +} + +// CreateHook validates and persists a new hook + revision #1. +func (h *Handler) CreateHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return + } + + var spec automation.HookSpec + if err := json.NewDecoder(r.Body).Decode(&spec); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + author, ok := h.resolveHookAuthor(w, r, userID, workspaceID) + if !ok { + return + } + + result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusCreated, hookToResponse(result)) +} + +// ListHooks returns every non-archived hook in the workspace. +func (h *Handler) ListHooks(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return + } + hooks, err := h.HookService.ListHooks(r.Context(), workspaceUUID) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]HookResponse, 0, len(hooks)) + for _, hook := range hooks { + resp = append(resp, hookToResponse(hook)) + } + writeJSON(w, http.StatusOK, resp) +} + +// GetHook returns one hook with its active revision. +func (h *Handler) GetHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + result, err := h.HookService.GetHook(r.Context(), workspaceUUID, hookUUID) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// UpdateHook appends a new revision and repoints the active pointer. +func (h *Handler) UpdateHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + var spec automation.HookSpec + if err := json.NewDecoder(r.Body).Decode(&spec); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + author, ok := h.resolveHookAuthor(w, r, userID, workspaceID) + if !ok { + return + } + result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// EnableHook re-enables a disabled hook. +func (h *Handler) EnableHook(w http.ResponseWriter, r *http.Request) { + h.setHookEnabled(w, r, true) +} + +// DisableHook disables a hook with an optional reason. +func (h *Handler) DisableHook(w http.ResponseWriter, r *http.Request) { + h.setHookEnabled(w, r, false) +} + +func (h *Handler) setHookEnabled(w http.ResponseWriter, r *http.Request, enabled bool) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + reason := "" + if !enabled && r.ContentLength != 0 { + var body struct { + Reason string `json:"reason"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + reason = body.Reason + } + result, err := h.HookService.SetEnabled(r.Context(), workspaceUUID, hookUUID, enabled, reason) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, hookToResponse(result)) +} + +// DeleteHook soft-archives a hook. +func (h *Handler) DeleteHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + if err := h.HookService.ArchiveHook(r.Context(), workspaceUUID, hookUUID); err != nil { + h.writeHookError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// ListHookExecutions returns the newest execution-trace rows for a hook. +func (h *Handler) ListHookExecutions(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) + if !ok { + return + } + execs, err := h.HookService.ListExecutions(r.Context(), workspaceUUID, hookUUID, hookExecutionListLimit) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]HookExecutionResponse, 0, len(execs)) + for _, e := range execs { + resp = append(resp, hookExecutionToResponse(e)) + } + writeJSON(w, http.StatusOK, resp) +} + +// hookPathParams resolves the workspace and the {id} hook UUID for a scoped route. +func (h *Handler) hookPathParams(w http.ResponseWriter, r *http.Request) (pgtype.UUID, pgtype.UUID, bool) { + workspaceID := h.resolveWorkspaceID(r) + workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id") + if !ok { + return pgtype.UUID{}, pgtype.UUID{}, false + } + hookUUID, ok := parseUUIDOrBadRequest(w, chi.URLParam(r, "id"), "id") + if !ok { + return pgtype.UUID{}, pgtype.UUID{}, false + } + return workspaceUUID, hookUUID, true +} + +// resolveHookAuthor derives the audit creator actor and the accountable human +// principal (§8). A member acts under their own authority; an agent must resolve +// to the human behind its current task, otherwise creation is refused — an agent +// UUID is never a substitute for a human authorization principal. +func (h *Handler) resolveHookAuthor(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, bool) { + actorType, actorID := h.resolveActor(r, userID, workspaceID) + actorUUID, err := util.ParseUUID(actorID) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid actor id") + return service.HookAuthor{}, false + } + principal := h.invokeOriginatorFromRequest(r, actorType, actorID) + if principal == "" { + h.writeDispatchBlocked(w, http.StatusForbidden, ReasonInvocationNotAllowed) + return service.HookAuthor{}, false + } + principalUUID, err := util.ParseUUID(principal) + if err != nil { + writeError(w, http.StatusForbidden, "no accountable authorization principal") + return service.HookAuthor{}, false + } + return service.HookAuthor{ActorType: actorType, ActorID: actorUUID, PrincipalUserID: principalUUID}, true +} + +func (h *Handler) writeHookError(w http.ResponseWriter, err error) { + if ve, ok := automation.AsValidationError(err); ok { + writeError(w, http.StatusBadRequest, ve.Error()) + return + } + switch { + case errors.Is(err, service.ErrHookNotFound): + writeError(w, http.StatusNotFound, "hook not found") + case errors.Is(err, service.ErrHookSystemManaged): + writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookNoPrincipal): + writeError(w, http.StatusForbidden, "no accountable authorization principal") + default: + writeError(w, http.StatusInternalServerError, "hook request failed") + } +} + +// rawJSON returns b as json.RawMessage, defaulting an empty column to a JSON null. +func rawJSON(b []byte) json.RawMessage { + if len(b) == 0 { + return json.RawMessage("null") + } + return json.RawMessage(b) +} + +// textValue unwraps a nullable text column to a plain string ("" when NULL). +func textValue(t pgtype.Text) string { + if t.Valid { + return t.String + } + return "" +} diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go new file mode 100644 index 00000000000..51bcbc810bf --- /dev/null +++ b/server/internal/handler/hook_test.go @@ -0,0 +1,240 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// enableHooksFlag flips automation_event_hooks on for the shared test handler and +// restores the previous flag service when the test ends. +func enableHooksFlag(t *testing.T) { + t.Helper() + prev := testHandler.FeatureFlags + p := featureflag.NewStaticProvider() + p.Set(featureflags.EventHooks, featureflag.Rule{Default: true}) + testHandler.FeatureFlags = featureflag.NewService(p) + t.Cleanup(func() { testHandler.FeatureFlags = prev }) +} + +// newMemberHookRequest builds a member-authenticated request (X-User-ID + +// X-Workspace-ID), the same identity a member hits the REST API with. +func newMemberHookRequest(method, path string, body any) *http.Request { + req := newJSONRequest(method, path, body) + req.Header.Set("X-User-ID", testUserID) + req.Header.Set("X-Workspace-ID", testWorkspaceID) + return req +} + +func newJSONRequest(method, path string, body any) *http.Request { + var r *http.Request + if body == nil { + r = httptest.NewRequest(method, path, nil) + } else { + buf, _ := json.Marshal(body) + r = httptest.NewRequest(method, path, bytes.NewReader(buf)) + } + r.Header.Set("Content-Type", "application/json") + return r +} + +// A minimal valid per_event spec; action targets are arbitrary uuids (PR2 +// validates uuid shape, not existence — matching/execution is a later slice). +func sampleHookSpec(name, message string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{ + "event": "issue.status_changed", + "match": map[string]any{"to": "done"}, + }, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{ + map[string]any{"type": "add_comment", "issue_id": "55555555-5555-5555-5555-555555555555", "message": message}, + }, + } +} + +func TestHookCRUDLifecycle(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + + // Create → revision #1. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("lifecycle hook", "first"))) + if w.Code != http.StatusCreated { + t.Fatalf("create: status %d: %s", w.Code, w.Body.String()) + } + var created HookResponse + if err := json.NewDecoder(w.Body).Decode(&created); err != nil { + t.Fatalf("decode create: %v", err) + } + if created.ID == "" || created.Revision.Revision != 1 || !created.Enabled { + t.Fatalf("unexpected create response: %+v", created) + } + if created.Revision.Event != "issue.status_changed" || created.Revision.FireMode != "per_event" { + t.Errorf("revision fields wrong: %+v", created.Revision) + } + hookID := created.ID + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM hook_revision WHERE hook_id = $1`, hookID) + testPool.Exec(context.Background(), `DELETE FROM hook WHERE id = $1`, hookID) + }) + + // Get. + w = httptest.NewRecorder() + testHandler.GetHook(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("get: status %d: %s", w.Code, w.Body.String()) + } + + // Update → new immutable revision #2, active pointer moves, name updated. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hookID, sampleHookSpec("renamed hook", "second")), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("update: status %d: %s", w.Code, w.Body.String()) + } + var updated HookResponse + json.NewDecoder(w.Body).Decode(&updated) + if updated.Revision.Revision != 2 || updated.Name != "renamed hook" { + t.Fatalf("update did not append revision / rename: %+v", updated) + } + if updated.Revision.ID == created.Revision.ID { + t.Errorf("update must create a NEW revision id, got same %s", updated.Revision.ID) + } + // The original revision row is retained (immutable history). + var revCount int + testPool.QueryRow(ctx, `SELECT count(*) FROM hook_revision WHERE hook_id = $1`, hookID).Scan(&revCount) + if revCount != 2 { + t.Errorf("hook_revision count = %d, want 2 (revisions are immutable, never overwritten)", revCount) + } + + // List includes it. + w = httptest.NewRecorder() + testHandler.ListHooks(w, newMemberHookRequest(http.MethodGet, "/api/hooks", nil)) + if w.Code != http.StatusOK { + t.Fatalf("list: status %d", w.Code) + } + var list []HookResponse + json.NewDecoder(w.Body).Decode(&list) + if !containsHook(list, hookID) { + t.Errorf("list does not contain created hook %s", hookID) + } + + // Disable → enabled=false with reason. + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hookID+"/disable", map[string]any{"reason": "paused"}), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("disable: status %d: %s", w.Code, w.Body.String()) + } + var disabled HookResponse + json.NewDecoder(w.Body).Decode(&disabled) + if disabled.Enabled || disabled.DisabledReason != "paused" { + t.Errorf("disable did not take: %+v", disabled) + } + + // Enable again. + w = httptest.NewRecorder() + testHandler.EnableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hookID+"/enable", nil), "id", hookID)) + var reenabled HookResponse + json.NewDecoder(w.Body).Decode(&reenabled) + if !reenabled.Enabled { + t.Errorf("enable did not take: %+v", reenabled) + } + + // Executions trace is empty (no matcher runs in PR2) but the endpoint works. + w = httptest.NewRecorder() + testHandler.ListHookExecutions(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID+"/executions", nil), "id", hookID)) + if w.Code != http.StatusOK { + t.Fatalf("executions: status %d", w.Code) + } + var execs []HookExecutionResponse + json.NewDecoder(w.Body).Decode(&execs) + if len(execs) != 0 { + t.Errorf("executions = %d, want 0 in store-only PR2", len(execs)) + } + + // Delete (soft archive) → subsequent get 404s. + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newMemberHookRequest(http.MethodDelete, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusNoContent { + t.Fatalf("delete: status %d: %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.GetHook(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID, nil), "id", hookID)) + if w.Code != http.StatusNotFound { + t.Errorf("get after archive: status %d, want 404", w.Code) + } + // The row is soft-archived, not physically deleted. + var archived bool + testPool.QueryRow(ctx, `SELECT archived_at IS NOT NULL FROM hook WHERE id = $1`, hookID).Scan(&archived) + if !archived { + t.Errorf("hook should be soft-archived, not deleted") + } +} + +// The whole surface is invisible unless the feature flag is on. +func TestHookRequiresFeatureFlag(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + prev := testHandler.FeatureFlags + testHandler.FeatureFlags = nil // nil service → every flag resolves to its default (off) + t.Cleanup(func() { testHandler.FeatureFlags = prev }) + + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("blocked", "x"))) + if w.Code != http.StatusNotFound { + t.Errorf("create with flag off: status %d, want 404", w.Code) + } +} + +// A bad spec is rejected at the API boundary with 400, never reaching the store. +func TestHookCreateRejectsInvalidSpec(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + bad := sampleHookSpec("bad", "x") + bad["do"] = []any{map[string]any{"type": "set_issue_status_many"}} // system-only + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", bad)) + if w.Code != http.StatusBadRequest { + t.Errorf("create with system-only action: status %d, want 400", w.Code) + } +} + +// An agent author with no resolvable human principal (§8) is refused. +func TestHookAgentRequiresPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + req := newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("agent hook", "x")) + // Trusted agent identity (task_token), valid agent uuid, but no X-Task-ID means + // no originator can be resolved → no accountable principal. + req.Header.Set("X-Actor-Source", "task_token") + req.Header.Set("X-Agent-ID", "66666666-6666-6666-6666-666666666666") + w := httptest.NewRecorder() + testHandler.CreateHook(w, req) + if w.Code != http.StatusForbidden { + t.Errorf("agent create without principal: status %d, want 403", w.Code) + } +} + +func containsHook(list []HookResponse, id string) bool { + for _, h := range list { + if h.ID == id { + return true + } + } + return false +} diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go new file mode 100644 index 00000000000..dc06bb7e327 --- /dev/null +++ b/server/internal/service/hook.go @@ -0,0 +1,344 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Hook CRUD errors surfaced to the handler for status mapping. Validation +// problems flow through as automation.ValidationError (→ 400). +var ( + ErrHookNotFound = errors.New("hook not found") + ErrHookSystemManaged = errors.New("system-managed hooks cannot be modified through this API") + ErrHookNoPrincipal = errors.New("no accountable authorization principal for this hook") +) + +// HookAuthor carries the resolved identity for a create/update: who is acting +// (creator, pure audit) and the accountable human whose authority the hook runs +// under (§8). An agent author must resolve to a real member principal. +type HookAuthor struct { + ActorType string // member | agent + ActorID pgtype.UUID + PrincipalUserID pgtype.UUID +} + +// HookWithRevision pairs a hook row with its active revision so the handler can +// render one complete view. The service returns db rows; the handler shapes JSON. +type HookWithRevision struct { + Hook db.Hook + Revision db.HookRevision +} + +// HookService is the store-only policy layer for Event Hooks (MUL-4332 PR2). +// It validates and persists hook specifications and their immutable revisions; +// it performs no matching or execution. Behaviour is gated at the handler by the +// automation_event_hooks feature flag, so creating hooks changes nothing at +// runtime until the executor slice ships and the flag is enabled. +type HookService struct { + Queries *db.Queries + TxStarter TxStarter +} + +func NewHookService(q *db.Queries, tx TxStarter) *HookService { + return &HookService{Queries: q, TxStarter: tx} +} + +// CreateHook validates the spec, resolves scope + principal, and inserts the +// hook together with revision #1 in one transaction. The two rows reference +// each other, so both ids are generated up front. +func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { + if err := automation.Validate(spec); err != nil { + return HookWithRevision{}, err + } + if !author.PrincipalUserID.Valid { + return HookWithRevision{}, ErrHookNoPrincipal + } + scopeType, scopeID, err := resolveScope(spec.Scope) + if err != nil { + return HookWithRevision{}, err + } + match, conditions, actions, err := marshalRevisionConfig(spec) + if err != nil { + return HookWithRevision{}, err + } + + hookID := util.NewUUID() + revisionID := util.NewUUID() + + var out HookWithRevision + err = s.inTx(ctx, func(qtx *db.Queries) error { + hook, err := qtx.CreateHook(ctx, db.CreateHookParams{ + ID: hookID, + WorkspaceID: workspaceID, + Name: spec.Name, + Enabled: true, + ActiveRevisionID: revisionID, + ScopeType: scopeType, + ScopeID: scopeID, + Origin: "user", + CreatorActorType: author.ActorType, + CreatorActorID: author.ActorID, + AuthorizationPrincipalUserID: author.PrincipalUserID, + }) + if err != nil { + return err + } + rev, err := qtx.CreateHookRevision(ctx, db.CreateHookRevisionParams{ + ID: revisionID, + HookID: hookID, + Revision: 1, + EventType: spec.When.Event, + Match: match, + Conditions: conditions, + FireMode: spec.Fire.Mode, + Actions: actions, + CreatedByType: author.ActorType, + CreatedByID: author.ActorID, + }) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + if err != nil { + return HookWithRevision{}, err + } + return out, nil +} + +// UpdateHook appends a new immutable revision from the spec and repoints the +// hook's active revision (§5.1). Existing executions stay pinned to their +// original revision. System-managed hooks are not editable here. +func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { + if err := automation.Validate(spec); err != nil { + return HookWithRevision{}, err + } + // Scope is immutable after creation; UpdateHook only replaces the revision + // config and the display name. + match, conditions, actions, err := marshalRevisionConfig(spec) + if err != nil { + return HookWithRevision{}, err + } + + existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return HookWithRevision{}, ErrHookNotFound + } + return HookWithRevision{}, err + } + if existing.ArchivedAt.Valid { + return HookWithRevision{}, ErrHookNotFound + } + if existing.Origin == "system" { + return HookWithRevision{}, ErrHookSystemManaged + } + + revisionID := util.NewUUID() + var out HookWithRevision + err = s.inTx(ctx, func(qtx *db.Queries) error { + maxRev, err := qtx.GetMaxHookRevision(ctx, hookID) + if err != nil { + return err + } + rev, err := qtx.CreateHookRevision(ctx, db.CreateHookRevisionParams{ + ID: revisionID, + HookID: hookID, + Revision: maxRev + 1, + EventType: spec.When.Event, + Match: match, + Conditions: conditions, + FireMode: spec.Fire.Mode, + Actions: actions, + CreatedByType: author.ActorType, + CreatedByID: author.ActorID, + }) + if err != nil { + return err + } + hook, err := qtx.SetHookActiveRevision(ctx, db.SetHookActiveRevisionParams{ + ID: hookID, + WorkspaceID: workspaceID, + ActiveRevisionID: revisionID, + Name: spec.Name, + }) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + if err != nil { + return HookWithRevision{}, err + } + return out, nil +} + +// GetHook loads a hook and its active revision. +func (s *HookService) GetHook(ctx context.Context, workspaceID, hookID pgtype.UUID) (HookWithRevision, error) { + hook, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return HookWithRevision{}, ErrHookNotFound + } + return HookWithRevision{}, err + } + if hook.ArchivedAt.Valid { + return HookWithRevision{}, ErrHookNotFound + } + rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return HookWithRevision{}, err + } + return HookWithRevision{Hook: hook, Revision: rev}, nil +} + +// ListHooks returns every non-archived hook in the workspace with its active +// revision. Hook counts per workspace are small (guardrails cap fan-out, not the +// number of rules), so the per-hook revision lookup is acceptable. +func (s *HookService) ListHooks(ctx context.Context, workspaceID pgtype.UUID) ([]HookWithRevision, error) { + hooks, err := s.Queries.ListHooksByWorkspace(ctx, workspaceID) + if err != nil { + return nil, err + } + out := make([]HookWithRevision, 0, len(hooks)) + for _, hook := range hooks { + rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return nil, err + } + out = append(out, HookWithRevision{Hook: hook, Revision: rev}) + } + return out, nil +} + +// SetEnabled enables/disables a hook. Disable only blocks future matches; it +// does not cancel queued/running executions (§5.1). +func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype.UUID, enabled bool, reason string) (HookWithRevision, error) { + existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return HookWithRevision{}, ErrHookNotFound + } + return HookWithRevision{}, err + } + if existing.ArchivedAt.Valid { + return HookWithRevision{}, ErrHookNotFound + } + if existing.Origin == "system" { + // Enabling/disabling a system hook is an admin-only lifecycle op reserved + // for the PR5 system-hook management path, not this user CRUD API. + return HookWithRevision{}, ErrHookSystemManaged + } + disabledReason := pgtype.Text{} + if !enabled && reason != "" { + disabledReason = pgtype.Text{String: reason, Valid: true} + } + hook, err := s.Queries.SetHookEnabled(ctx, db.SetHookEnabledParams{ + ID: hookID, + WorkspaceID: workspaceID, + Enabled: enabled, + DisabledReason: disabledReason, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return HookWithRevision{}, ErrHookNotFound + } + return HookWithRevision{}, err + } + rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return HookWithRevision{}, err + } + return HookWithRevision{Hook: hook, Revision: rev}, nil +} + +// ArchiveHook soft-deletes a hook (§5.1); revisions/executions/effects are kept. +func (s *HookService) ArchiveHook(ctx context.Context, workspaceID, hookID pgtype.UUID) error { + existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrHookNotFound + } + return err + } + if existing.ArchivedAt.Valid { + return ErrHookNotFound + } + if existing.Origin == "system" { + return ErrHookSystemManaged + } + if _, err := s.Queries.ArchiveHook(ctx, db.ArchiveHookParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrHookNotFound + } + return err + } + return nil +} + +// ListExecutions returns the newest execution-trace rows for a hook (bounded). +func (s *HookService) ListExecutions(ctx context.Context, workspaceID, hookID pgtype.UUID, limit int32) ([]db.HookExecution, error) { + // Confirm the hook belongs to the workspace before exposing its trace. + if _, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrHookNotFound + } + return nil, err + } + return s.Queries.ListHookExecutionsByHook(ctx, db.ListHookExecutionsByHookParams{HookID: hookID, Limit: limit}) +} + +func (s *HookService) inTx(ctx context.Context, fn func(qtx *db.Queries) error) error { + tx, err := s.TxStarter.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + if err := fn(s.Queries.WithTx(tx)); err != nil { + return err + } + return tx.Commit(ctx) +} + +// resolveScope maps the optional scope spec to (scope_type, scope_id). The spec +// has already passed automation.Validate, so an issue scope always has a valid id. +func resolveScope(scope *automation.ScopeSpec) (string, pgtype.UUID, error) { + if scope == nil || scope.Type == automation.ScopeWorkspace { + return automation.ScopeWorkspace, pgtype.UUID{}, nil + } + id, err := util.ParseUUID(scope.ID) + if err != nil { + return "", pgtype.UUID{}, err + } + return automation.ScopeIssue, id, nil +} + +// marshalRevisionConfig produces the JSONB payloads stored on a revision. +// conditions and actions are always stored as (possibly empty) arrays; match as +// an object. +func marshalRevisionConfig(spec automation.HookSpec) (match, conditions, actions []byte, err error) { + match = spec.When.Match + if len(match) == 0 { + match = []byte("{}") + } + if len(spec.If) == 0 { + conditions = []byte("[]") + } else if conditions, err = json.Marshal(spec.If); err != nil { + return nil, nil, nil, err + } + if actions, err = json.Marshal(spec.Do); err != nil { + return nil, nil, nil, err + } + return match, conditions, actions, nil +} diff --git a/server/internal/util/pgx.go b/server/internal/util/pgx.go index 5b344067c6a..10d7ec1fe09 100644 --- a/server/internal/util/pgx.go +++ b/server/internal/util/pgx.go @@ -5,9 +5,17 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +// NewUUID mints a fresh random (v4) pgtype.UUID. Use for app-generated primary +// keys — e.g. two related rows that must reference each other before either is +// inserted, so neither can rely on a DB DEFAULT round-trip. +func NewUUID() pgtype.UUID { + return pgtype.UUID{Bytes: uuid.New(), Valid: true} +} + // ParseUUID parses s into a pgtype.UUID. Invalid input returns an error // instead of a zero-valued UUID — silently dropping bad input has caused // data-loss bugs (e.g. DELETE matching no rows, returning 204 success). diff --git a/server/pkg/db/generated/hook.sql.go b/server/pkg/db/generated/hook.sql.go new file mode 100644 index 00000000000..82d4fab9ff3 --- /dev/null +++ b/server/pkg/db/generated/hook.sql.go @@ -0,0 +1,502 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: hook.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const archiveHook = `-- name: ArchiveHook :one +UPDATE hook SET + archived_at = now(), + enabled = false, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type ArchiveHookParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Soft archive (DELETE). Existing revisions / executions / effects are retained +// for audit; the hook is also disabled so nothing can match it. +func (q *Queries) ArchiveHook(ctx context.Context, arg ArchiveHookParams) (Hook, error) { + row := q.db.QueryRow(ctx, archiveHook, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const createHook = `-- name: CreateHook :one + +INSERT INTO hook ( + id, workspace_id, name, enabled, active_revision_id, + scope_type, scope_id, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, + $9, $10, $11 +) +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type CreateHookParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` + Origin string `json:"origin"` + CreatorActorType string `json:"creator_actor_type"` + CreatorActorID pgtype.UUID `json:"creator_actor_id"` + AuthorizationPrincipalUserID pgtype.UUID `json:"authorization_principal_user_id"` +} + +// Event Hooks MVP (MUL-4332) — hook / revision / execution persistence access. +// All associations are application-validated; there are no foreign keys. Every +// write is workspace-scoped for tenant isolation. The hook and its immutable +// revisions are created together in one transaction with app-generated ids +// (hook.active_revision_id and hook_revision.id are chosen up front) because +// the two rows reference each other and there is no FK to order them. +func (q *Queries) CreateHook(ctx context.Context, arg CreateHookParams) (Hook, error) { + row := q.db.QueryRow(ctx, createHook, + arg.ID, + arg.WorkspaceID, + arg.Name, + arg.Enabled, + arg.ActiveRevisionID, + arg.ScopeType, + arg.ScopeID, + arg.Origin, + arg.CreatorActorType, + arg.CreatorActorID, + arg.AuthorizationPrincipalUserID, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const createHookRevision = `-- name: CreateHookRevision :one +INSERT INTO hook_revision ( + id, hook_id, revision, event_type, match, conditions, fire_mode, actions, + created_by_type, created_by_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, + $9, $10 +) +RETURNING id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at +` + +type CreateHookRevisionParams struct { + ID pgtype.UUID `json:"id"` + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` + EventType string `json:"event_type"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions []byte `json:"actions"` + CreatedByType string `json:"created_by_type"` + CreatedByID pgtype.UUID `json:"created_by_id"` +} + +func (q *Queries) CreateHookRevision(ctx context.Context, arg CreateHookRevisionParams) (HookRevision, error) { + row := q.db.QueryRow(ctx, createHookRevision, + arg.ID, + arg.HookID, + arg.Revision, + arg.EventType, + arg.Match, + arg.Conditions, + arg.FireMode, + arg.Actions, + arg.CreatedByType, + arg.CreatedByID, + ) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + +const getHookInWorkspace = `-- name: GetHookInWorkspace :one +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE id = $1 AND workspace_id = $2 +` + +type GetHookInWorkspaceParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +func (q *Queries) GetHookInWorkspace(ctx context.Context, arg GetHookInWorkspaceParams) (Hook, error) { + row := q.db.QueryRow(ctx, getHookInWorkspace, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const getHookRevision = `-- name: GetHookRevision :one +SELECT id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at FROM hook_revision +WHERE id = $1 +` + +func (q *Queries) GetHookRevision(ctx context.Context, id pgtype.UUID) (HookRevision, error) { + row := q.db.QueryRow(ctx, getHookRevision, id) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + +const getMaxHookRevision = `-- name: GetMaxHookRevision :one +SELECT COALESCE(MAX(revision), 0)::int AS max_revision +FROM hook_revision +WHERE hook_id = $1 +` + +// Highest revision number for a hook, 0 when none exist yet. Used to compute the +// next revision on PATCH. +func (q *Queries) GetMaxHookRevision(ctx context.Context, hookID pgtype.UUID) (int32, error) { + row := q.db.QueryRow(ctx, getMaxHookRevision, hookID) + var max_revision int32 + err := row.Scan(&max_revision) + return max_revision, err +} + +const listHookExecutionsByHook = `-- name: ListHookExecutionsByHook :many +SELECT id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at FROM hook_execution +WHERE hook_id = $1 +ORDER BY created_at DESC +LIMIT $2 +` + +type ListHookExecutionsByHookParams struct { + HookID pgtype.UUID `json:"hook_id"` + Limit int32 `json:"limit"` +} + +// Execution trace for the debug/explain endpoints. Newest first, bounded. +func (q *Queries) ListHookExecutionsByHook(ctx context.Context, arg ListHookExecutionsByHookParams) ([]HookExecution, error) { + rows, err := q.db.Query(ctx, listHookExecutionsByHook, arg.HookID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []HookExecution{} + for rows.Next() { + var i HookExecution + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listHooksByScope = `-- name: ListHooksByScope :many +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE workspace_id = $1 AND scope_type = $2 AND scope_id = $3 AND archived_at IS NULL +ORDER BY created_at DESC +` + +type ListHooksByScopeParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` +} + +func (q *Queries) ListHooksByScope(ctx context.Context, arg ListHooksByScopeParams) ([]Hook, error) { + rows, err := q.db.Query(ctx, listHooksByScope, arg.WorkspaceID, arg.ScopeType, arg.ScopeID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Hook{} + for rows.Next() { + var i Hook + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listHooksByWorkspace = `-- name: ListHooksByWorkspace :many +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE workspace_id = $1 AND archived_at IS NULL +ORDER BY created_at DESC +` + +func (q *Queries) ListHooksByWorkspace(ctx context.Context, workspaceID pgtype.UUID) ([]Hook, error) { + rows, err := q.db.Query(ctx, listHooksByWorkspace, workspaceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Hook{} + for rows.Next() { + var i Hook + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setHookActiveRevision = `-- name: SetHookActiveRevision :one +UPDATE hook SET + active_revision_id = $3, + name = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type SetHookActiveRevisionParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + Name string `json:"name"` +} + +// Switch the active revision pointer and update the display name (PATCH). +// Revisions themselves are never mutated; a config change appends a new revision +// and repoints here. Scope is immutable after creation and is not touched. +func (q *Queries) SetHookActiveRevision(ctx context.Context, arg SetHookActiveRevisionParams) (Hook, error) { + row := q.db.QueryRow(ctx, setHookActiveRevision, + arg.ID, + arg.WorkspaceID, + arg.ActiveRevisionID, + arg.Name, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + +const setHookEnabled = `-- name: SetHookEnabled :one +UPDATE hook SET + enabled = $3, + disabled_reason = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at +` + +type SetHookEnabledParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Enabled bool `json:"enabled"` + DisabledReason pgtype.Text `json:"disabled_reason"` +} + +func (q *Queries) SetHookEnabled(ctx context.Context, arg SetHookEnabledParams) (Hook, error) { + row := q.db.QueryRow(ctx, setHookEnabled, + arg.ID, + arg.WorkspaceID, + arg.Enabled, + arg.DisabledReason, + ) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} diff --git a/server/pkg/db/generated/models.go b/server/pkg/db/generated/models.go index 4efa7f9332f..1a75488f0c2 100644 --- a/server/pkg/db/generated/models.go +++ b/server/pkg/db/generated/models.go @@ -173,6 +173,16 @@ type Attachment struct { TaskID pgtype.UUID `json:"task_id"` } +// Durable row-lockable automation state, not an audit log. No foreign keys. +type AutomationState struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` + State []byte `json:"state"` + Version int64 `json:"version"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` +} + type Autopilot struct { ID pgtype.UUID `json:"id"` WorkspaceID pgtype.UUID `json:"workspace_id"` @@ -558,6 +568,85 @@ type GithubPullRequestCheckSuite struct { UpdatedAt pgtype.Timestamptz `json:"updated_at"` } +// Event Hooks stable identity and lifecycle owner. No foreign keys; application validates all associations. +type Hook struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + ActiveRevisionID pgtype.UUID `json:"active_revision_id"` + ScopeType string `json:"scope_type"` + ScopeID pgtype.UUID `json:"scope_id"` + RetireAfterEventSeq pgtype.Int8 `json:"retire_after_event_seq"` + Origin string `json:"origin"` + SystemKey pgtype.Text `json:"system_key"` + SystemVersion pgtype.Int4 `json:"system_version"` + CreatorActorType string `json:"creator_actor_type"` + CreatorActorID pgtype.UUID `json:"creator_actor_id"` + AuthorizationPrincipalUserID pgtype.UUID `json:"authorization_principal_user_id"` + DisabledReason pgtype.Text `json:"disabled_reason"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + ArchivedAt pgtype.Timestamptz `json:"archived_at"` +} + +// One durable idempotency record per hook execution action. No foreign keys. +type HookActionEffect struct { + ID pgtype.UUID `json:"id"` + EffectKey string `json:"effect_key"` + ExecutionID pgtype.UUID `json:"execution_id"` + ActionIndex int32 `json:"action_index"` + ActionType string `json:"action_type"` + Status string `json:"status"` + ResolvedInput []byte `json:"resolved_input"` + OutputType pgtype.Text `json:"output_type"` + OutputID pgtype.UUID `json:"output_id"` + Attempts int32 `json:"attempts"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + CompletedAt pgtype.Timestamptz `json:"completed_at"` +} + +// Durable matcher/executor trace with revision pinning. No foreign keys; application validates all associations. +type HookExecution struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + HookID pgtype.UUID `json:"hook_id"` + HookRevisionID pgtype.UUID `json:"hook_revision_id"` + EventID pgtype.UUID `json:"event_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + Status string `json:"status"` + SkipReason pgtype.Text `json:"skip_reason"` + MatchSnapshot []byte `json:"match_snapshot"` + ConditionSnapshot []byte `json:"condition_snapshot"` + CurrentActionIndex int32 `json:"current_action_index"` + Attempts int32 `json:"attempts"` + NextAttemptAt pgtype.Timestamptz `json:"next_attempt_at"` + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseExpiresAt pgtype.Timestamptz `json:"lease_expires_at"` + ErrorCode pgtype.Text `json:"error_code"` + Error pgtype.Text `json:"error"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + StartedAt pgtype.Timestamptz `json:"started_at"` + CompletedAt pgtype.Timestamptz `json:"completed_at"` +} + +// Immutable Event Hooks configuration revisions. No foreign keys; application validates hook ownership. +type HookRevision struct { + ID pgtype.UUID `json:"id"` + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` + EventType string `json:"event_type"` + Match []byte `json:"match"` + Conditions []byte `json:"conditions"` + FireMode string `json:"fire_mode"` + Actions []byte `json:"actions"` + CreatedByType string `json:"created_by_type"` + CreatedByID pgtype.UUID `json:"created_by_id"` + CreatedAt pgtype.Timestamptz `json:"created_at"` +} + type InboxItem struct { ID pgtype.UUID `json:"id"` WorkspaceID pgtype.UUID `json:"workspace_id"` diff --git a/server/pkg/db/queries/hook.sql b/server/pkg/db/queries/hook.sql new file mode 100644 index 00000000000..a4497332a47 --- /dev/null +++ b/server/pkg/db/queries/hook.sql @@ -0,0 +1,89 @@ +-- Event Hooks MVP (MUL-4332) — hook / revision / execution persistence access. +-- All associations are application-validated; there are no foreign keys. Every +-- write is workspace-scoped for tenant isolation. The hook and its immutable +-- revisions are created together in one transaction with app-generated ids +-- (hook.active_revision_id and hook_revision.id are chosen up front) because +-- the two rows reference each other and there is no FK to order them. + +-- name: CreateHook :one +INSERT INTO hook ( + id, workspace_id, name, enabled, active_revision_id, + scope_type, scope_id, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, + $9, $10, $11 +) +RETURNING *; + +-- name: GetHookInWorkspace :one +SELECT * FROM hook +WHERE id = $1 AND workspace_id = $2; + +-- name: ListHooksByWorkspace :many +SELECT * FROM hook +WHERE workspace_id = $1 AND archived_at IS NULL +ORDER BY created_at DESC; + +-- name: ListHooksByScope :many +SELECT * FROM hook +WHERE workspace_id = $1 AND scope_type = $2 AND scope_id = $3 AND archived_at IS NULL +ORDER BY created_at DESC; + +-- name: SetHookActiveRevision :one +-- Switch the active revision pointer and update the display name (PATCH). +-- Revisions themselves are never mutated; a config change appends a new revision +-- and repoints here. Scope is immutable after creation and is not touched. +UPDATE hook SET + active_revision_id = $3, + name = $4, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: SetHookEnabled :one +UPDATE hook SET + enabled = $3, + disabled_reason = sqlc.narg('disabled_reason'), + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: ArchiveHook :one +-- Soft archive (DELETE). Existing revisions / executions / effects are retained +-- for audit; the hook is also disabled so nothing can match it. +UPDATE hook SET + archived_at = now(), + enabled = false, + updated_at = now() +WHERE id = $1 AND workspace_id = $2 AND archived_at IS NULL +RETURNING *; + +-- name: CreateHookRevision :one +INSERT INTO hook_revision ( + id, hook_id, revision, event_type, match, conditions, fire_mode, actions, + created_by_type, created_by_id +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, + $9, $10 +) +RETURNING *; + +-- name: GetHookRevision :one +SELECT * FROM hook_revision +WHERE id = $1; + +-- name: GetMaxHookRevision :one +-- Highest revision number for a hook, 0 when none exist yet. Used to compute the +-- next revision on PATCH. +SELECT COALESCE(MAX(revision), 0)::int AS max_revision +FROM hook_revision +WHERE hook_id = $1; + +-- name: ListHookExecutionsByHook :many +-- Execution trace for the debug/explain endpoints. Newest first, bounded. +SELECT * FROM hook_execution +WHERE hook_id = $1 +ORDER BY created_at DESC +LIMIT $2; From 06f6b85ca582c534b18fc29575e8263e8c5b1e4e Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 19:10:24 +0800 Subject: [PATCH 08/15] fix(automation): close PR2 review must-fixes on hook policy layer (MUL-4332 PR2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon's four must-fixes on the hook CRUD layer. All four become real authorization / correctness issues once PR3 starts consuming saved rules. 1. [P0] Edit authorization gate. Any workspace member could PATCH/enable/disable/ delete any user hook while it kept running under the original principal (authorization-principal substitution). Now only the hook's original authorization principal OR a workspace owner/admin may modify it; the principal is NOT transferred on edit. UpdateHook/SetEnabled/ArchiveHook enforce authorizeHookEdit. (Deny-by-default per the proposed semantic, pending Bohan's final confirmation; easy to switch to principal-transfer.) 2. [P0] Fail-closed target/workspace validation. Create/update now validate, in the write transaction under the current principal, that every referenced target exists in the hook's workspace: issue scope, condition issue ids, and action issue/agent/member/autopilot targets — with the trigger_agent target also checked for archived/runtime and canInvokeAgent admission. The resolved principal must still be a workspace member. A missing target now yields 400, not a persisted 201 (§13). Adds GetMemberInWorkspace. 3. [P1] Strict schema. The handler decodes with DisallowUnknownFields (rejects unknown top-level/nested keys); the validator rejects per-action disallowed-but-known fields (e.g. agent_id on add_comment), validates the issue-status enum on set_issue_status / issues_status / issue_field(status), and bounds id-set and message sizes. 4. [P1] Concurrent PATCH. UpdateHook now SELECT ... FOR UPDATE locks the hook row and re-checks archived/origin/authorization inside the lock before allocating MAX(revision)+1, so concurrent edits serialize instead of colliding on idx_hook_revision_unique (was 23505 → 500). Adds GetHookForUpdate. Tests: strict-schema rejections (unknown field, disallowed action field, bad status enum), fail-closed targets (nonexistent issue/member/agent/autopilot → 400; real invokable agent → 201), edit authorization (non-principal non-admin → 403 on patch/disable/delete; principal and owner/admin → 200; principal never transferred), and 8-way concurrent PATCH asserting contiguous revisions 1..9 with no error. dry-run/explain + event-correlation queries are re-sliced to PR3 (they need the matcher's evaluation), pending Bohan's confirmation. Co-authored-by: multica-agent --- server/internal/automation/schema.go | 17 + server/internal/automation/validate.go | 90 ++++- server/internal/automation/validate_test.go | 11 + server/internal/handler/hook.go | 93 +++-- server/internal/handler/hook_test.go | 377 ++++++++++++++++---- server/internal/service/hook.go | 271 +++++++++++--- server/pkg/db/generated/hook.sql.go | 41 +++ server/pkg/db/generated/member.sql.go | 26 ++ server/pkg/db/queries/hook.sql | 9 + server/pkg/db/queries/member.sql | 7 + 10 files changed, 799 insertions(+), 143 deletions(-) diff --git a/server/internal/automation/schema.go b/server/internal/automation/schema.go index 2ad09857701..61f300bf783 100644 --- a/server/internal/automation/schema.go +++ b/server/internal/automation/schema.go @@ -135,6 +135,23 @@ var validIssueFields = map[string]bool{ IssueFieldParentIssueID: true, } +// validIssueStatuses mirrors the issue.status DB CHECK (migrations/001) and the +// handler's validIssueStatuses. A status-valued condition or action is rejected +// unless its value is one of these, so a hook can never persist an unreachable +// status (MUL-4332 PR2 review point 3). +var validIssueStatuses = map[string]bool{ + "backlog": true, + "todo": true, + "in_progress": true, + "in_review": true, + "done": true, + "blocked": true, + "cancelled": true, +} + +// isValidIssueStatus reports whether s is a persistable issue status. +func isValidIssueStatus(s string) bool { return validIssueStatuses[s] } + // conditionDependencyEvent maps a condition to the single v1 domain event that // can change its truth value. A rising_edge hook must listen to exactly that // event so its latch can be re-evaluated (§5.2). In the v1 fixed vocabulary diff --git a/server/internal/automation/validate.go b/server/internal/automation/validate.go index ff712e3c329..61b911bf5ef 100644 --- a/server/internal/automation/validate.go +++ b/server/internal/automation/validate.go @@ -12,6 +12,9 @@ import ( const ( MaxActionsPerHook = 8 MaxNameLength = 200 + MaxConditionIDs = 100 + MaxMatchSetSize = 100 + MaxMessageLength = 4000 ) // ValidationError is a user-fixable problem with a hook spec. Handlers map it to @@ -20,6 +23,14 @@ type ValidationError struct{ msg string } func (e *ValidationError) Error() string { return e.msg } +// NewValidationError builds a ValidationError. Exported so callers that extend +// author-time validation with checks this pure package cannot do (e.g. the +// service's workspace-scoped target existence checks) surface the same typed +// error and share the handler's single 400 mapping. +func NewValidationError(format string, args ...any) *ValidationError { + return &ValidationError{msg: fmt.Sprintf(format, args...)} +} + func verr(format string, args ...any) error { return &ValidationError{msg: fmt.Sprintf(format, args...)} } @@ -117,6 +128,9 @@ func validateMatch(raw []byte, schema EventSchema) error { } func validateClauseValues(field string, kind FieldKind, clause MatchClause) error { + if clause.Op == MatchIn && len(clause.Set) > MaxMatchSetSize { + return verr("match field %q has %d values, at most %d allowed", field, len(clause.Set), MaxMatchSetSize) + } if kind != FieldUUID { return nil // string fields accept any scalar; existence needs no value check } @@ -156,6 +170,9 @@ func validateIssuesStatus(c IssuesStatusCond) error { if len(c.IDs) == 0 { return verr("issues_status.ids must not be empty") } + if len(c.IDs) > MaxConditionIDs { + return verr("issues_status.ids has %d ids, at most %d allowed", len(c.IDs), MaxConditionIDs) + } for _, id := range c.IDs { if !validUUID(id) { return verr("issues_status.ids must be uuids, got %q", id) @@ -165,6 +182,13 @@ func validateIssuesStatus(c IssuesStatusCond) error { if hasAll == hasAny { return verr("exactly one of issues_status.all or issues_status.any must be set") } + status := c.All + if hasAny { + status = c.Any + } + if !isValidIssueStatus(status) { + return verr("issues_status status %q is not a valid issue status", status) + } return nil } @@ -179,8 +203,19 @@ func validateIssueField(c IssueFieldCond) error { if hasEq == hasIn { return verr("exactly one of issue_field.eq or issue_field.in must be set") } - // Field-typed values: status is a free string; the id-shaped fields require uuids. - if c.Field != IssueFieldStatus { + if len(c.In) > MaxConditionIDs { + return verr("issue_field.in has %d values, at most %d allowed", len(c.In), MaxConditionIDs) + } + switch c.Field { + case IssueFieldStatus: + // status is a free string but must be a persistable issue status. + for _, v := range collectValues(c.Eq, c.In) { + if !isValidIssueStatus(v) { + return verr("issue_field status %q is not a valid issue status", v) + } + } + default: + // id-shaped fields (assignee_id / parent_issue_id) require uuids. if hasEq && !validUUID(c.Eq) { return verr("issue_field.eq must be a uuid for field %q", c.Field) } @@ -193,6 +228,16 @@ func validateIssueField(c IssueFieldCond) error { return nil } +// collectValues returns eq (if set) plus the in slice as one list. +func collectValues(eq string, in []string) []string { + out := make([]string, 0, len(in)+1) + if eq != "" { + out = append(out, eq) + } + out = append(out, in...) + return out +} + func validateFire(spec HookSpec) error { switch spec.Fire.Mode { case FirePerEvent: @@ -225,6 +270,18 @@ func validateRisingEdgeCoverage(spec HookSpec) error { return nil } +// actionAllowedFields declares the exact set of ActionSpec parameter fields each +// action type may set. Any other non-empty field is rejected so a stray param +// (e.g. an agent_id smuggled onto add_comment) can never be persisted onto a +// revision (MUL-4332 PR2 review point 3). +var actionAllowedFields = map[string]map[string]bool{ + ActionSetIssueStatus: {"issue_id": true, "status": true}, + ActionTriggerAgent: {"issue_id": true, "agent_id": true}, + ActionAddComment: {"issue_id": true, "message": true}, + ActionSendInbox: {"member_id": true, "message": true}, + ActionRunAutopilot: {"autopilot_id": true}, +} + func validateAction(a ActionSpec) error { if systemActionTypes[a.Type] { return verr("action type %q is reserved for system hooks", a.Type) @@ -232,6 +289,12 @@ func validateAction(a ActionSpec) error { if !userActionTypes[a.Type] { return verr("unknown action type %q", a.Type) } + if err := rejectUnexpectedActionFields(a); err != nil { + return err + } + if a.Message != "" && len(a.Message) > MaxMessageLength { + return verr("%s message must be at most %d characters", a.Type, MaxMessageLength) + } switch a.Type { case ActionSetIssueStatus: if !validUUID(a.IssueID) { @@ -240,6 +303,9 @@ func validateAction(a ActionSpec) error { if a.Status == "" { return verr("set_issue_status requires status") } + if !isValidIssueStatus(a.Status) { + return verr("set_issue_status status %q is not a valid issue status", a.Status) + } case ActionTriggerAgent: if !validUUID(a.IssueID) { return verr("trigger_agent requires a valid issue_id") @@ -269,6 +335,26 @@ func validateAction(a ActionSpec) error { return nil } +// rejectUnexpectedActionFields fails if the action sets any parameter field not +// allowed for its type — strict "exactly the allowed fields" enforcement. +func rejectUnexpectedActionFields(a ActionSpec) error { + allowed := actionAllowedFields[a.Type] + present := map[string]string{ + "issue_id": a.IssueID, + "status": a.Status, + "agent_id": a.AgentID, + "member_id": a.MemberID, + "message": a.Message, + "autopilot_id": a.AutopilotID, + } + for field, value := range present { + if value != "" && !allowed[field] { + return verr("%s does not accept field %q", a.Type, field) + } + } + return nil +} + func validUUID(s string) bool { if s == "" { return false diff --git a/server/internal/automation/validate_test.go b/server/internal/automation/validate_test.go index ce6d3839b5f..ca8258eadee 100644 --- a/server/internal/automation/validate_test.go +++ b/server/internal/automation/validate_test.go @@ -79,6 +79,17 @@ func TestValidateRejectsInvalidSpecs(t *testing.T) { {"set_issue_status missing status", func(s *HookSpec) { s.Do = []ActionSpec{{Type: ActionSetIssueStatus, IssueID: uuidC}} }, "requires status"}, + {"set_issue_status invalid status enum", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionSetIssueStatus, IssueID: uuidC, Status: "ascended"}} + }, "not a valid issue status"}, + {"add_comment with disallowed agent_id field", func(s *HookSpec) { + s.Do = []ActionSpec{{Type: ActionAddComment, IssueID: uuidC, Message: "hi", AgentID: uuidA}} + }, "does not accept field"}, + {"issues_status invalid status enum", func(s *HookSpec) { + s.When = WhenSpec{Event: "issue.status_changed"} + s.Fire = FireSpec{Mode: FirePerEvent} + s.If = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA}, All: "ascended"}}} + }, "not a valid issue status"}, {"trigger_agent bad agent", func(s *HookSpec) { s.Do = []ActionSpec{{Type: ActionTriggerAgent, IssueID: uuidC, AgentID: "x"}} }, "valid agent_id"}, diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go index 51a9ed2ddc0..e52c388ede7 100644 --- a/server/internal/handler/hook.go +++ b/server/internal/handler/hook.go @@ -129,18 +129,17 @@ func (h *Handler) CreateHook(w http.ResponseWriter, r *http.Request) { return } - var spec automation.HookSpec - if err := json.NewDecoder(r.Body).Decode(&spec); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") + spec, ok := decodeHookSpec(w, r) + if !ok { return } - author, ok := h.resolveHookAuthor(w, r, userID, workspaceID) + author, canInvoke, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } - result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author) + result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author, canInvoke) if err != nil { h.writeHookError(w, err) return @@ -201,16 +200,15 @@ func (h *Handler) UpdateHook(w http.ResponseWriter, r *http.Request) { if !ok { return } - var spec automation.HookSpec - if err := json.NewDecoder(r.Body).Decode(&spec); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") + spec, ok := decodeHookSpec(w, r) + if !ok { return } - author, ok := h.resolveHookAuthor(w, r, userID, workspaceID) + author, canInvoke, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } - result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author) + result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author, canInvoke) if err != nil { h.writeHookError(w, err) return @@ -232,6 +230,11 @@ func (h *Handler) setHookEnabled(w http.ResponseWriter, r *http.Request, enabled if !h.hookEnabled(w, r) { return } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) if !ok { return @@ -247,7 +250,11 @@ func (h *Handler) setHookEnabled(w http.ResponseWriter, r *http.Request, enabled } reason = body.Reason } - result, err := h.HookService.SetEnabled(r.Context(), workspaceUUID, hookUUID, enabled, reason) + author, _, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + result, err := h.HookService.SetEnabled(r.Context(), workspaceUUID, hookUUID, enabled, reason, author) if err != nil { h.writeHookError(w, err) return @@ -260,11 +267,20 @@ func (h *Handler) DeleteHook(w http.ResponseWriter, r *http.Request) { if !h.hookEnabled(w, r) { return } + workspaceID := h.resolveWorkspaceID(r) + userID, ok := requireUserID(w, r) + if !ok { + return + } workspaceUUID, hookUUID, ok := h.hookPathParams(w, r) if !ok { return } - if err := h.HookService.ArchiveHook(r.Context(), workspaceUUID, hookUUID); err != nil { + author, _, ok := h.resolveHookWriter(w, r, userID, workspaceID) + if !ok { + return + } + if err := h.HookService.ArchiveHook(r.Context(), workspaceUUID, hookUUID, author); err != nil { h.writeHookError(w, err) return } @@ -306,28 +322,61 @@ func (h *Handler) hookPathParams(w http.ResponseWriter, r *http.Request) (pgtype return workspaceUUID, hookUUID, true } -// resolveHookAuthor derives the audit creator actor and the accountable human -// principal (§8). A member acts under their own authority; an agent must resolve -// to the human behind its current task, otherwise creation is refused — an agent -// UUID is never a substitute for a human authorization principal. -func (h *Handler) resolveHookAuthor(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, bool) { +// decodeHookSpec strictly decodes the request body into a HookSpec, rejecting +// unknown fields so a stray top-level or nested key can never be silently +// accepted (MUL-4332 PR2 review point 3). Per-action disallowed fields are +// rejected by the automation validator. +func decodeHookSpec(w http.ResponseWriter, r *http.Request) (automation.HookSpec, bool) { + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + var spec automation.HookSpec + if err := dec.Decode(&spec); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error()) + return automation.HookSpec{}, false + } + return spec, true +} + +// resolveHookWriter derives the audit creator actor, the accountable human +// principal (§8), and whether that principal is a workspace owner/admin, plus the +// agent-admission callback used for fail-closed trigger_agent validation. A +// member acts under their own authority; an agent must resolve to the human +// behind its current task. The resolved principal must still be a workspace +// member (review point 2) — an agent UUID is never a substitute for a human, and +// a departed principal can no longer author hooks. +func (h *Handler) resolveHookWriter(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, service.CanInvokeAgent, bool) { actorType, actorID := h.resolveActor(r, userID, workspaceID) actorUUID, err := util.ParseUUID(actorID) if err != nil { writeError(w, http.StatusBadRequest, "invalid actor id") - return service.HookAuthor{}, false + return service.HookAuthor{}, nil, false } principal := h.invokeOriginatorFromRequest(r, actorType, actorID) if principal == "" { h.writeDispatchBlocked(w, http.StatusForbidden, ReasonInvocationNotAllowed) - return service.HookAuthor{}, false + return service.HookAuthor{}, nil, false } principalUUID, err := util.ParseUUID(principal) if err != nil { writeError(w, http.StatusForbidden, "no accountable authorization principal") - return service.HookAuthor{}, false + return service.HookAuthor{}, nil, false + } + // The accountable principal must still be a member of this workspace. + member, err := h.getWorkspaceMember(r.Context(), principal, workspaceID) + if err != nil { + writeError(w, http.StatusForbidden, "authorization principal is not a member of this workspace") + return service.HookAuthor{}, nil, false } - return service.HookAuthor{ActorType: actorType, ActorID: actorUUID, PrincipalUserID: principalUUID}, true + author := service.HookAuthor{ + ActorType: actorType, + ActorID: actorUUID, + PrincipalUserID: principalUUID, + IsWorkspaceAdmin: roleAllowed(member.Role, "owner", "admin"), + } + canInvoke := func(agent db.Agent) bool { + return h.canInvokeAgent(r.Context(), agent, actorType, actorID, principal, workspaceID) + } + return author, canInvoke, true } func (h *Handler) writeHookError(w http.ResponseWriter, err error) { @@ -340,6 +389,8 @@ func (h *Handler) writeHookError(w http.ResponseWriter, err error) { writeError(w, http.StatusNotFound, "hook not found") case errors.Is(err, service.ErrHookSystemManaged): writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookForbidden): + writeError(w, http.StatusForbidden, err.Error()) case errors.Is(err, service.ErrHookNoPrincipal): writeError(w, http.StatusForbidden, "no accountable authorization principal") default: diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go index 51bcbc810bf..df68606f36d 100644 --- a/server/internal/handler/hook_test.go +++ b/server/internal/handler/hook_test.go @@ -4,14 +4,28 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" + "sync" + "sync/atomic" "testing" + "github.com/multica-ai/multica/server/internal/automation" "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" "github.com/multica-ai/multica/server/pkg/featureflag" ) +// hookSpecFromMap round-trips a test spec map through JSON into a typed HookSpec, +// for tests that drive the service layer directly. +func hookSpecFromMap(m map[string]any) automation.HookSpec { + buf, _ := json.Marshal(m) + var spec automation.HookSpec + json.Unmarshal(buf, &spec) + return spec +} + // enableHooksFlag flips automation_event_hooks on for the shared test handler and // restores the previous flag service when the test ends. func enableHooksFlag(t *testing.T) { @@ -23,11 +37,14 @@ func enableHooksFlag(t *testing.T) { t.Cleanup(func() { testHandler.FeatureFlags = prev }) } -// newMemberHookRequest builds a member-authenticated request (X-User-ID + -// X-Workspace-ID), the same identity a member hits the REST API with. func newMemberHookRequest(method, path string, body any) *http.Request { + return newUserHookRequest(method, path, body, testUserID) +} + +// newUserHookRequest builds a member-authenticated request for the given user. +func newUserHookRequest(method, path string, body any, userID string) *http.Request { req := newJSONRequest(method, path, body) - req.Header.Set("X-User-ID", testUserID) + req.Header.Set("X-User-ID", userID) req.Header.Set("X-Workspace-ID", testWorkspaceID) return req } @@ -44,9 +61,74 @@ func newJSONRequest(method, path string, body any) *http.Request { return r } -// A minimal valid per_event spec; action targets are arbitrary uuids (PR2 -// validates uuid shape, not existence — matching/execution is a later slice). -func sampleHookSpec(name, message string) map[string]any { +// seedHookIssue inserts a real issue in the test workspace and returns its id. +func seedHookIssue(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO issue (workspace_id, title, status, priority, creator_type, creator_id, number) + VALUES ($1, 'hook target issue', 'todo', 'medium', 'member', $2, + COALESCE((SELECT MAX(number) FROM issue WHERE workspace_id = $1), 0) + 1) + RETURNING id`, testWorkspaceID, testUserID).Scan(&id); err != nil { + t.Fatalf("seed issue: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM issue WHERE id = $1`, id) }) + return id +} + +// seededHookAgentID returns the workspace-visible agent seeded by the fixture. +func seededHookAgentID(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), + `SELECT id FROM agent WHERE workspace_id = $1 AND name = 'Handler Test Agent' LIMIT 1`, + testWorkspaceID).Scan(&id); err != nil { + t.Fatalf("load seeded agent: %v", err) + } + return id +} + +// testOwnerMemberID returns the member row id of the fixture owner (testUserID). +func testOwnerMemberID(t *testing.T) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), + `SELECT id FROM member WHERE workspace_id = $1 AND user_id = $2`, + testWorkspaceID, testUserID).Scan(&id); err != nil { + t.Fatalf("load owner member: %v", err) + } + return id +} + +// hookSeedCounter makes each seeded user's email unique across calls. +var hookSeedCounter atomic.Int64 + +// seedHookMember inserts a user + workspace member with the given role and +// returns the user id. +func seedHookMember(t *testing.T, role string) string { + t.Helper() + ctx := context.Background() + var userID string + email := fmt.Sprintf("hook-%s-%d-%d@test.local", role, hookSeedCounter.Add(1), len(t.Name())) + if err := testPool.QueryRow(ctx, + `INSERT INTO "user" (name, email) VALUES ($1, $2) RETURNING id`, + "Hook "+role, email).Scan(&userID); err != nil { + t.Fatalf("seed user: %v", err) + } + if _, err := testPool.Exec(ctx, + `INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, $3)`, + testWorkspaceID, userID, role); err != nil { + t.Fatalf("seed member: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, testWorkspaceID, userID) + testPool.Exec(context.Background(), `DELETE FROM "user" WHERE id = $1`, userID) + }) + return userID +} + +// sampleHookSpec is a minimal valid per_event spec: comment an existing issue. +func sampleHookSpec(name, message, issueID string) map[string]any { return map[string]any{ "name": name, "when": map[string]any{ @@ -55,42 +137,45 @@ func sampleHookSpec(name, message string) map[string]any { }, "fire": map[string]any{"mode": "per_event"}, "do": []any{ - map[string]any{"type": "add_comment", "issue_id": "55555555-5555-5555-5555-555555555555", "message": message}, + map[string]any{"type": "add_comment", "issue_id": issueID, "message": message}, }, } } +func createHookAs(t *testing.T, userID string, spec map[string]any) HookResponse { + t.Helper() + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", spec, userID)) + if w.Code != http.StatusCreated { + t.Fatalf("create hook as %s: status %d: %s", userID, w.Code, w.Body.String()) + } + var resp HookResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode create: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM hook_revision WHERE hook_id = $1`, resp.ID) + testPool.Exec(context.Background(), `DELETE FROM hook WHERE id = $1`, resp.ID) + }) + return resp +} + func TestHookCRUDLifecycle(t *testing.T) { if testPool == nil { t.Skip("database unavailable") } enableHooksFlag(t) ctx := context.Background() + issueID := seedHookIssue(t) - // Create → revision #1. - w := httptest.NewRecorder() - testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("lifecycle hook", "first"))) - if w.Code != http.StatusCreated { - t.Fatalf("create: status %d: %s", w.Code, w.Body.String()) - } - var created HookResponse - if err := json.NewDecoder(w.Body).Decode(&created); err != nil { - t.Fatalf("decode create: %v", err) - } - if created.ID == "" || created.Revision.Revision != 1 || !created.Enabled { + created := createHookAs(t, testUserID, sampleHookSpec("lifecycle hook", "first", issueID)) + if created.Revision.Revision != 1 || !created.Enabled || created.Revision.Event != "issue.status_changed" { t.Fatalf("unexpected create response: %+v", created) } - if created.Revision.Event != "issue.status_changed" || created.Revision.FireMode != "per_event" { - t.Errorf("revision fields wrong: %+v", created.Revision) - } hookID := created.ID - t.Cleanup(func() { - testPool.Exec(context.Background(), `DELETE FROM hook_revision WHERE hook_id = $1`, hookID) - testPool.Exec(context.Background(), `DELETE FROM hook WHERE id = $1`, hookID) - }) // Get. - w = httptest.NewRecorder() + w := httptest.NewRecorder() testHandler.GetHook(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID, nil), "id", hookID)) if w.Code != http.StatusOK { t.Fatalf("get: status %d: %s", w.Code, w.Body.String()) @@ -98,31 +183,24 @@ func TestHookCRUDLifecycle(t *testing.T) { // Update → new immutable revision #2, active pointer moves, name updated. w = httptest.NewRecorder() - testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hookID, sampleHookSpec("renamed hook", "second")), "id", hookID)) + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hookID, sampleHookSpec("renamed hook", "second", issueID)), "id", hookID)) if w.Code != http.StatusOK { t.Fatalf("update: status %d: %s", w.Code, w.Body.String()) } var updated HookResponse json.NewDecoder(w.Body).Decode(&updated) - if updated.Revision.Revision != 2 || updated.Name != "renamed hook" { - t.Fatalf("update did not append revision / rename: %+v", updated) - } - if updated.Revision.ID == created.Revision.ID { - t.Errorf("update must create a NEW revision id, got same %s", updated.Revision.ID) + if updated.Revision.Revision != 2 || updated.Name != "renamed hook" || updated.Revision.ID == created.Revision.ID { + t.Fatalf("update did not append a new revision / rename: %+v", updated) } - // The original revision row is retained (immutable history). var revCount int testPool.QueryRow(ctx, `SELECT count(*) FROM hook_revision WHERE hook_id = $1`, hookID).Scan(&revCount) if revCount != 2 { - t.Errorf("hook_revision count = %d, want 2 (revisions are immutable, never overwritten)", revCount) + t.Errorf("hook_revision count = %d, want 2 (revisions are immutable)", revCount) } - // List includes it. + // List. w = httptest.NewRecorder() testHandler.ListHooks(w, newMemberHookRequest(http.MethodGet, "/api/hooks", nil)) - if w.Code != http.StatusOK { - t.Fatalf("list: status %d", w.Code) - } var list []HookResponse json.NewDecoder(w.Body).Decode(&list) if !containsHook(list, hookID) { @@ -132,9 +210,6 @@ func TestHookCRUDLifecycle(t *testing.T) { // Disable → enabled=false with reason. w = httptest.NewRecorder() testHandler.DisableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hookID+"/disable", map[string]any{"reason": "paused"}), "id", hookID)) - if w.Code != http.StatusOK { - t.Fatalf("disable: status %d: %s", w.Code, w.Body.String()) - } var disabled HookResponse json.NewDecoder(w.Body).Decode(&disabled) if disabled.Enabled || disabled.DisabledReason != "paused" { @@ -150,17 +225,12 @@ func TestHookCRUDLifecycle(t *testing.T) { t.Errorf("enable did not take: %+v", reenabled) } - // Executions trace is empty (no matcher runs in PR2) but the endpoint works. + // Executions trace is empty in store-only PR2 but the endpoint works. w = httptest.NewRecorder() testHandler.ListHookExecutions(w, withURLParam(newMemberHookRequest(http.MethodGet, "/api/hooks/"+hookID+"/executions", nil), "id", hookID)) if w.Code != http.StatusOK { t.Fatalf("executions: status %d", w.Code) } - var execs []HookExecutionResponse - json.NewDecoder(w.Body).Decode(&execs) - if len(execs) != 0 { - t.Errorf("executions = %d, want 0 in store-only PR2", len(execs)) - } // Delete (soft archive) → subsequent get 404s. w = httptest.NewRecorder() @@ -173,15 +243,8 @@ func TestHookCRUDLifecycle(t *testing.T) { if w.Code != http.StatusNotFound { t.Errorf("get after archive: status %d, want 404", w.Code) } - // The row is soft-archived, not physically deleted. - var archived bool - testPool.QueryRow(ctx, `SELECT archived_at IS NOT NULL FROM hook WHERE id = $1`, hookID).Scan(&archived) - if !archived { - t.Errorf("hook should be soft-archived, not deleted") - } } -// The whole surface is invisible unless the feature flag is on. func TestHookRequiresFeatureFlag(t *testing.T) { if testPool == nil { t.Skip("database unavailable") @@ -191,24 +254,211 @@ func TestHookRequiresFeatureFlag(t *testing.T) { t.Cleanup(func() { testHandler.FeatureFlags = prev }) w := httptest.NewRecorder() - testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("blocked", "x"))) + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("blocked", "x", "55555555-5555-5555-5555-555555555555"))) if w.Code != http.StatusNotFound { t.Errorf("create with flag off: status %d, want 404", w.Code) } } -// A bad spec is rejected at the API boundary with 400, never reaching the store. -func TestHookCreateRejectsInvalidSpec(t *testing.T) { +// Strict schema: unknown fields and per-action disallowed fields and bad status +// are all rejected at the boundary with 400, never persisted (review point 3). +func TestHookStrictSchemaRejections(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + + cases := map[string]func() map[string]any{ + "unknown top-level field": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["unexpected"] = true + return s + }, + "disallowed action field": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "add_comment", "issue_id": issueID, "message": "hi", "agent_id": "66666666-6666-6666-6666-666666666666"}} + return s + }, + "invalid status enum": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "set_issue_status", "issue_id": issueID, "status": "ascended"}} + return s + }, + "system-only action": func() map[string]any { + s := sampleHookSpec("x", "hi", issueID) + s["do"] = []any{map[string]any{"type": "set_issue_status_many"}} + return s + }, + } + for name, build := range cases { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", build())) + if w.Code != http.StatusBadRequest { + t.Errorf("status %d, want 400: %s", w.Code, w.Body.String()) + } + }) + } +} + +// Fail-closed target validation: a spec that references a target absent from the +// workspace is rejected with 400, not persisted (review point 2). +func TestHookFailClosedTargets(t *testing.T) { if testPool == nil { t.Skip("database unavailable") } enableHooksFlag(t) - bad := sampleHookSpec("bad", "x") - bad["do"] = []any{map[string]any{"type": "set_issue_status_many"}} // system-only + realIssue := seedHookIssue(t) + const ghost = "77777777-7777-7777-7777-777777777777" + + action := func(a map[string]any) map[string]any { + s := sampleHookSpec("targets", "hi", realIssue) + s["do"] = []any{a} + return s + } + cases := map[string]map[string]any{ + "nonexistent issue": action(map[string]any{"type": "add_comment", "issue_id": ghost, "message": "hi"}), + "nonexistent member": action(map[string]any{"type": "send_inbox", "member_id": ghost, "message": "hi"}), + "nonexistent agent": action(map[string]any{"type": "trigger_agent", "issue_id": realIssue, "agent_id": ghost}), + "nonexistent autopilot": action(map[string]any{"type": "run_autopilot", "autopilot_id": ghost}), + } + for name, spec := range cases { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", spec)) + if w.Code != http.StatusBadRequest { + t.Errorf("status %d, want 400 (fail-closed): %s", w.Code, w.Body.String()) + } + }) + } + + // A real, invokable agent target is accepted. + t.Run("real invokable agent accepted", func(t *testing.T) { + spec := action(map[string]any{"type": "trigger_agent", "issue_id": realIssue, "agent_id": seededHookAgentID(t)}) + createHookAs(t, testUserID, spec) // fails the test if not 201 + }) +} + +// Only the hook's principal or a workspace owner/admin may edit it; an arbitrary +// member cannot rewrite a rule that keeps running under someone else's authority +// (review point 1). +func TestHookEditAuthorization(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + + author := seedHookMember(t, "member") // principal, non-admin + other := seedHookMember(t, "member") // non-principal, non-admin + + hook := createHookAs(t, author, sampleHookSpec("owned by author", "hi", issueID)) + + // A different non-admin member cannot edit / disable / delete it. + patch := withURLParam(newUserHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("hijacked", "x", issueID), other), "id", hook.ID) w := httptest.NewRecorder() - testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", bad)) - if w.Code != http.StatusBadRequest { - t.Errorf("create with system-only action: status %d, want 400", w.Code) + testHandler.UpdateHook(w, patch) + if w.Code != http.StatusForbidden { + t.Errorf("other member PATCH: status %d, want 403", w.Code) + } + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newUserHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/disable", nil, other), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("other member disable: status %d, want 403", w.Code) + } + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newUserHookRequest(http.MethodDelete, "/api/hooks/"+hook.ID, nil, other), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("other member delete: status %d, want 403", w.Code) + } + + // The principal can edit their own hook. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newUserHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("by principal", "x", issueID), author), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("principal PATCH: status %d, want 200: %s", w.Code, w.Body.String()) + } + + // A workspace owner/admin (the fixture owner) can edit any hook. + w = httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("by admin", "x", issueID)), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("admin PATCH: status %d, want 200: %s", w.Code, w.Body.String()) + } + + // The principal was NOT transferred by any edit — it is still the author. + var principal string + testPool.QueryRow(context.Background(), + `SELECT authorization_principal_user_id FROM hook WHERE id = $1`, hook.ID).Scan(&principal) + if principal != author { + t.Errorf("principal = %s, want %s (edits must not transfer the principal)", principal, author) + } +} + +// Concurrent PATCHes on the same hook must each append a distinct, contiguous +// revision without a MAX+1 unique-index collision surfacing as a 500 (review +// point 4). Driven at the service layer so all workers hit the pool concurrently. +func TestHookConcurrentPatchAppendsContiguousRevisions(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + hook := createHookAs(t, testUserID, sampleHookSpec("concurrent", "hi", issueID)) + + admin := service.HookAuthor{ + ActorType: "member", + ActorID: parseUUID(testUserID), + PrincipalUserID: parseUUID(testUserID), + IsWorkspaceAdmin: true, + } + hookUUID := parseUUID(hook.ID) + wsUUID := parseUUID(testWorkspaceID) + + const workers = 8 + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + <-start + spec := hookSpecFromMap(sampleHookSpec(fmt.Sprintf("rev-%d", n), "x", issueID)) + _, err := testHandler.HookService.UpdateHook(ctx, wsUUID, hookUUID, spec, admin, nil) + errs <- err + }(i) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent PATCH errored (revision race not serialized): %v", err) + } + } + + // Revisions must be exactly 1..(workers+1), contiguous and unique. + rows, err := testPool.Query(ctx, `SELECT revision FROM hook_revision WHERE hook_id = $1 ORDER BY revision`, hook.ID) + if err != nil { + t.Fatalf("query revisions: %v", err) + } + defer rows.Close() + var revs []int + for rows.Next() { + var r int + rows.Scan(&r) + revs = append(revs, r) + } + if len(revs) != workers+1 { + t.Fatalf("revision count = %d, want %d", len(revs), workers+1) + } + for i, r := range revs { + if r != i+1 { + t.Fatalf("revisions not contiguous at index %d: got %d, want %d (revs=%v)", i, r, i+1, revs) + } } } @@ -218,7 +468,8 @@ func TestHookAgentRequiresPrincipal(t *testing.T) { t.Skip("database unavailable") } enableHooksFlag(t) - req := newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("agent hook", "x")) + issueID := seedHookIssue(t) + req := newMemberHookRequest(http.MethodPost, "/api/hooks", sampleHookSpec("agent hook", "x", issueID)) // Trusted agent identity (task_token), valid agent uuid, but no X-Task-ID means // no originator can be resolved → no accountable principal. req.Header.Set("X-Actor-Source", "task_token") diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go index dc06bb7e327..b0fa44fccf9 100644 --- a/server/internal/service/hook.go +++ b/server/internal/service/hook.go @@ -15,22 +15,32 @@ import ( ) // Hook CRUD errors surfaced to the handler for status mapping. Validation -// problems flow through as automation.ValidationError (→ 400). +// problems (shape or unresolvable target) flow through as +// *automation.ValidationError (→ 400). var ( ErrHookNotFound = errors.New("hook not found") ErrHookSystemManaged = errors.New("system-managed hooks cannot be modified through this API") ErrHookNoPrincipal = errors.New("no accountable authorization principal for this hook") + ErrHookForbidden = errors.New("only the hook's principal or a workspace admin may modify it") ) -// HookAuthor carries the resolved identity for a create/update: who is acting -// (creator, pure audit) and the accountable human whose authority the hook runs -// under (§8). An agent author must resolve to a real member principal. +// HookAuthor carries the resolved identity for a hook write: who is acting +// (creator, pure audit), the accountable human whose authority the hook runs +// under (§8), and whether that human is a workspace owner/admin. An agent author +// must resolve to a real member principal. type HookAuthor struct { - ActorType string // member | agent - ActorID pgtype.UUID - PrincipalUserID pgtype.UUID + ActorType string // member | agent + ActorID pgtype.UUID + PrincipalUserID pgtype.UUID + IsWorkspaceAdmin bool } +// CanInvokeAgent is the admission predicate the handler supplies (wrapping +// Handler.canInvokeAgent) so the service can fail-closed on a trigger_agent +// target without importing request context. A nil predicate denies every +// trigger_agent target (fail-closed). +type CanInvokeAgent func(agent db.Agent) bool + // HookWithRevision pairs a hook row with its active revision so the handler can // render one complete view. The service returns db rows; the handler shapes JSON. type HookWithRevision struct { @@ -52,10 +62,11 @@ func NewHookService(q *db.Queries, tx TxStarter) *HookService { return &HookService{Queries: q, TxStarter: tx} } -// CreateHook validates the spec, resolves scope + principal, and inserts the -// hook together with revision #1 in one transaction. The two rows reference -// each other, so both ids are generated up front. -func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { +// CreateHook validates the spec (shape + workspace-scoped targets), resolves +// scope + principal, and inserts the hook together with revision #1 in one +// transaction. The two rows reference each other, so both ids are generated up +// front. +func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor, canInvoke CanInvokeAgent) (HookWithRevision, error) { if err := automation.Validate(spec); err != nil { return HookWithRevision{}, err } @@ -76,6 +87,9 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { + if err := validateTargets(ctx, qtx, workspaceID, spec, canInvoke); err != nil { + return err + } hook, err := qtx.CreateHook(ctx, db.CreateHookParams{ ID: hookID, WorkspaceID: workspaceID, @@ -117,36 +131,41 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s } // UpdateHook appends a new immutable revision from the spec and repoints the -// hook's active revision (§5.1). Existing executions stay pinned to their -// original revision. System-managed hooks are not editable here. -func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { +// hook's active revision (§5.1). It locks the hook row first so concurrent +// PATCHes serialize and MAX(revision)+1 can never collide (review point 4), and +// re-checks archived/origin/authorization inside the lock. Only the hook's +// principal or a workspace admin may edit (review point 1). Scope is immutable. +func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor, canInvoke CanInvokeAgent) (HookWithRevision, error) { if err := automation.Validate(spec); err != nil { return HookWithRevision{}, err } - // Scope is immutable after creation; UpdateHook only replaces the revision - // config and the display name. match, conditions, actions, err := marshalRevisionConfig(spec) if err != nil { return HookWithRevision{}, err } - existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return HookWithRevision{}, ErrHookNotFound - } - return HookWithRevision{}, err - } - if existing.ArchivedAt.Valid { - return HookWithRevision{}, ErrHookNotFound - } - if existing.Origin == "system" { - return HookWithRevision{}, ErrHookSystemManaged - } - revisionID := util.NewUUID() var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { + existing, err := qtx.GetHookForUpdate(ctx, db.GetHookForUpdateParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrHookNotFound + } + return err + } + if existing.ArchivedAt.Valid { + return ErrHookNotFound + } + if existing.Origin == "system" { + return ErrHookSystemManaged + } + if err := authorizeHookEdit(existing, author); err != nil { + return err + } + if err := validateTargets(ctx, qtx, workspaceID, spec, canInvoke); err != nil { + return err + } maxRev, err := qtx.GetMaxHookRevision(ctx, hookID) if err != nil { return err @@ -223,23 +242,12 @@ func (s *HookService) ListHooks(ctx context.Context, workspaceID pgtype.UUID) ([ } // SetEnabled enables/disables a hook. Disable only blocks future matches; it -// does not cancel queued/running executions (§5.1). -func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype.UUID, enabled bool, reason string) (HookWithRevision, error) { - existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return HookWithRevision{}, ErrHookNotFound - } +// does not cancel queued/running executions (§5.1). Only the principal or a +// workspace admin may toggle it. +func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype.UUID, enabled bool, reason string, author HookAuthor) (HookWithRevision, error) { + if _, err := s.loadEditableHook(ctx, workspaceID, hookID, author); err != nil { return HookWithRevision{}, err } - if existing.ArchivedAt.Valid { - return HookWithRevision{}, ErrHookNotFound - } - if existing.Origin == "system" { - // Enabling/disabling a system hook is an admin-only lifecycle op reserved - // for the PR5 system-hook management path, not this user CRUD API. - return HookWithRevision{}, ErrHookSystemManaged - } disabledReason := pgtype.Text{} if !enabled && reason != "" { disabledReason = pgtype.Text{String: reason, Valid: true} @@ -264,20 +272,11 @@ func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype } // ArchiveHook soft-deletes a hook (§5.1); revisions/executions/effects are kept. -func (s *HookService) ArchiveHook(ctx context.Context, workspaceID, hookID pgtype.UUID) error { - existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return ErrHookNotFound - } +// Only the principal or a workspace admin may archive it. +func (s *HookService) ArchiveHook(ctx context.Context, workspaceID, hookID pgtype.UUID, author HookAuthor) error { + if _, err := s.loadEditableHook(ctx, workspaceID, hookID, author); err != nil { return err } - if existing.ArchivedAt.Valid { - return ErrHookNotFound - } - if existing.Origin == "system" { - return ErrHookSystemManaged - } if _, err := s.Queries.ArchiveHook(ctx, db.ArchiveHookParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { if errors.Is(err, pgx.ErrNoRows) { return ErrHookNotFound @@ -299,6 +298,47 @@ func (s *HookService) ListExecutions(ctx context.Context, workspaceID, hookID pg return s.Queries.ListHookExecutionsByHook(ctx, db.ListHookExecutionsByHookParams{HookID: hookID, Limit: limit}) } +// loadEditableHook loads a non-archived, non-system hook and enforces the edit +// authorization gate, for the enable/disable/archive paths. +func (s *HookService) loadEditableHook(ctx context.Context, workspaceID, hookID pgtype.UUID, author HookAuthor) (db.Hook, error) { + existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return db.Hook{}, ErrHookNotFound + } + return db.Hook{}, err + } + if existing.ArchivedAt.Valid { + return db.Hook{}, ErrHookNotFound + } + if existing.Origin == "system" { + return db.Hook{}, ErrHookSystemManaged + } + if err := authorizeHookEdit(existing, author); err != nil { + return db.Hook{}, err + } + return existing, nil +} + +// authorizeHookEdit implements the edit gate (review point 1): a workspace +// owner/admin may edit any hook; otherwise only the hook's original +// authorization principal may. The principal is NOT transferred on edit, so an +// arbitrary member can never rewrite a rule that keeps running under someone +// else's authority. +func authorizeHookEdit(hook db.Hook, author HookAuthor) error { + if author.IsWorkspaceAdmin { + return nil + } + if principalMatches(hook.AuthorizationPrincipalUserID, author.PrincipalUserID) { + return nil + } + return ErrHookForbidden +} + +func principalMatches(a, b pgtype.UUID) bool { + return a.Valid && b.Valid && a.Bytes == b.Bytes +} + func (s *HookService) inTx(ctx context.Context, fn func(qtx *db.Queries) error) error { tx, err := s.TxStarter.Begin(ctx) if err != nil { @@ -342,3 +382,120 @@ func marshalRevisionConfig(spec automation.HookSpec) (match, conditions, actions } return match, conditions, actions, nil } + +// validateTargets fail-closed checks that every id the spec references exists in +// the hook's workspace under the current principal (review point 2). §13 requires +// this at create/update time so an illegal configuration never enters the store +// and never reaches a worker. Uses the tx-bound queries so the checks share the +// write transaction. +func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, canInvoke CanInvokeAgent) error { + if spec.Scope != nil && spec.Scope.Type == automation.ScopeIssue { + if err := requireIssue(ctx, qtx, workspaceID, spec.Scope.ID, "scope.id"); err != nil { + return err + } + } + for i, cond := range spec.If { + if cond.IssuesStatus != nil { + for _, id := range cond.IssuesStatus.IDs { + if err := requireIssue(ctx, qtx, workspaceID, id, fmt.Sprintf("if[%d].issues_status.ids", i)); err != nil { + return err + } + } + } + if cond.IssueField != nil { + if err := requireIssue(ctx, qtx, workspaceID, cond.IssueField.ID, fmt.Sprintf("if[%d].issue_field.id", i)); err != nil { + return err + } + } + } + for i, action := range spec.Do { + if err := validateActionTargets(ctx, qtx, workspaceID, i, action, canInvoke); err != nil { + return err + } + } + return nil +} + +func validateActionTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, i int, a automation.ActionSpec, canInvoke CanInvokeAgent) error { + where := fmt.Sprintf("do[%d].%s", i, a.Type) + switch a.Type { + case automation.ActionSetIssueStatus, automation.ActionAddComment: + return requireIssue(ctx, qtx, workspaceID, a.IssueID, where+".issue_id") + case automation.ActionTriggerAgent: + if err := requireIssue(ctx, qtx, workspaceID, a.IssueID, where+".issue_id"); err != nil { + return err + } + return requireInvokableAgent(ctx, qtx, workspaceID, a.AgentID, where+".agent_id", canInvoke) + case automation.ActionSendInbox: + return requireMember(ctx, qtx, workspaceID, a.MemberID, where+".member_id") + case automation.ActionRunAutopilot: + return requireAutopilot(ctx, qtx, workspaceID, a.AutopilotID, where+".autopilot_id") + } + return nil +} + +func requireIssue(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := qtx.GetIssueInWorkspace(ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references issue %s which does not exist in this workspace", field, id) + } + return err + } + return nil +} + +func requireMember(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := qtx.GetMemberInWorkspace(ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references member %s which is not in this workspace", field, id) + } + return err + } + return nil +} + +func requireAutopilot(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + if _, err := qtx.GetAutopilotInWorkspace(ctx, db.GetAutopilotInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s which does not exist in this workspace", field, id) + } + return err + } + return nil +} + +// requireInvokableAgent confirms the agent exists in the workspace, is not +// archived, has a runtime, and is invokable by the current principal — the same +// admission the interactive trigger path enforces, applied fail-closed at save. +func requireInvokableAgent(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string, canInvoke CanInvokeAgent) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + agent, err := qtx.GetAgentInWorkspace(ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references agent %s which does not exist in this workspace", field, id) + } + return err + } + if agent.ArchivedAt.Valid || !agent.RuntimeID.Valid { + return automation.NewValidationError("%s references agent %s which is archived or has no runtime", field, id) + } + if canInvoke == nil || !canInvoke(agent) { + return automation.NewValidationError("%s references agent %s which the hook's principal may not invoke", field, id) + } + return nil +} diff --git a/server/pkg/db/generated/hook.sql.go b/server/pkg/db/generated/hook.sql.go index 82d4fab9ff3..7f51bf31fad 100644 --- a/server/pkg/db/generated/hook.sql.go +++ b/server/pkg/db/generated/hook.sql.go @@ -179,6 +179,47 @@ func (q *Queries) CreateHookRevision(ctx context.Context, arg CreateHookRevision return i, err } +const getHookForUpdate = `-- name: GetHookForUpdate :one +SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE +` + +type GetHookForUpdateParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Row-locking load used by PATCH so concurrent edits to the same hook serialize: +// the lock holder allocates the next revision and repoints the active pointer +// before the next waiter reads MAX(revision), so idx_hook_revision_unique can +// never be violated by a MAX+1 race (MUL-4332 PR2 review point 4). +func (q *Queries) GetHookForUpdate(ctx context.Context, arg GetHookForUpdateParams) (Hook, error) { + row := q.db.QueryRow(ctx, getHookForUpdate, arg.ID, arg.WorkspaceID) + var i Hook + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.Name, + &i.Enabled, + &i.ActiveRevisionID, + &i.ScopeType, + &i.ScopeID, + &i.RetireAfterEventSeq, + &i.Origin, + &i.SystemKey, + &i.SystemVersion, + &i.CreatorActorType, + &i.CreatorActorID, + &i.AuthorizationPrincipalUserID, + &i.DisabledReason, + &i.CreatedAt, + &i.UpdatedAt, + &i.ArchivedAt, + ) + return i, err +} + const getHookInWorkspace = `-- name: GetHookInWorkspace :one SELECT id, workspace_id, name, enabled, active_revision_id, scope_type, scope_id, retire_after_event_seq, origin, system_key, system_version, creator_actor_type, creator_actor_id, authorization_principal_user_id, disabled_reason, created_at, updated_at, archived_at FROM hook WHERE id = $1 AND workspace_id = $2 diff --git a/server/pkg/db/generated/member.sql.go b/server/pkg/db/generated/member.sql.go index ee18f5e15ad..322a22b0cfd 100644 --- a/server/pkg/db/generated/member.sql.go +++ b/server/pkg/db/generated/member.sql.go @@ -86,6 +86,32 @@ func (q *Queries) GetMemberByUserAndWorkspace(ctx context.Context, arg GetMember return i, err } +const getMemberInWorkspace = `-- name: GetMemberInWorkspace :one +SELECT id, workspace_id, user_id, role, created_at FROM member +WHERE id = $1 AND workspace_id = $2 +` + +type GetMemberInWorkspaceParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Workspace-scoped member existence check by member row id. Used to fail-closed +// validate a send_inbox action target belongs to the hook's workspace +// (MUL-4332 PR2 review point 2). +func (q *Queries) GetMemberInWorkspace(ctx context.Context, arg GetMemberInWorkspaceParams) (Member, error) { + row := q.db.QueryRow(ctx, getMemberInWorkspace, arg.ID, arg.WorkspaceID) + var i Member + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.UserID, + &i.Role, + &i.CreatedAt, + ) + return i, err +} + const listMembers = `-- name: ListMembers :many SELECT id, workspace_id, user_id, role, created_at FROM member WHERE workspace_id = $1 diff --git a/server/pkg/db/queries/hook.sql b/server/pkg/db/queries/hook.sql index a4497332a47..a9d0f4e26d1 100644 --- a/server/pkg/db/queries/hook.sql +++ b/server/pkg/db/queries/hook.sql @@ -21,6 +21,15 @@ RETURNING *; SELECT * FROM hook WHERE id = $1 AND workspace_id = $2; +-- name: GetHookForUpdate :one +-- Row-locking load used by PATCH so concurrent edits to the same hook serialize: +-- the lock holder allocates the next revision and repoints the active pointer +-- before the next waiter reads MAX(revision), so idx_hook_revision_unique can +-- never be violated by a MAX+1 race (MUL-4332 PR2 review point 4). +SELECT * FROM hook +WHERE id = $1 AND workspace_id = $2 +FOR UPDATE; + -- name: ListHooksByWorkspace :many SELECT * FROM hook WHERE workspace_id = $1 AND archived_at IS NULL diff --git a/server/pkg/db/queries/member.sql b/server/pkg/db/queries/member.sql index f93365d3b32..2c4e471eef9 100644 --- a/server/pkg/db/queries/member.sql +++ b/server/pkg/db/queries/member.sql @@ -7,6 +7,13 @@ ORDER BY created_at ASC; SELECT * FROM member WHERE id = $1; +-- name: GetMemberInWorkspace :one +-- Workspace-scoped member existence check by member row id. Used to fail-closed +-- validate a send_inbox action target belongs to the hook's workspace +-- (MUL-4332 PR2 review point 2). +SELECT * FROM member +WHERE id = $1 AND workspace_id = $2; + -- name: GetMemberByUserAndWorkspace :one SELECT * FROM member WHERE user_id = $1 AND workspace_id = $2; From 19d7e77e5bd0b267fbe814db2925dc588a5720dd Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 19:44:24 +0800 Subject: [PATCH 09/15] fix(automation): admission by stored principal + tx-scoped authz (MUL-4332 PR2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon's round-3 review on the hook policy layer. 1. Admission by the hook's STORED principal, not the editor. Previously resolveHookWriter captured the requester's principal in a canInvoke closure, so a workspace admin editing member A's hook could add a trigger_agent for the admin's own private agent — saved as A, who cannot invoke it. Target admission is now evaluated inside the write transaction against the principal read from the FOR UPDATE-locked hook row (the creator on create, the stored principal on update). New internal/admission package holds the pure agent/ autopilot predicates so handler and service share one source of truth. 2. run_autopilot now checks the stored principal's write permission (role/authorship/collaborator, reusing memberCanWriteAutopilot semantics via admission.AutopilotWriteByOwnership), not mere existence. issue_field parent_issue_id / assignee_id eq|in operands are now validated to resolve to workspace resources (issue, or member/agent for assignees). 3. Membership, role and the edit gate are re-derived INSIDE the write transaction: create re-checks the principal is a current member; update/ enable/disable/archive load the FOR UPDATE-locked row, re-read the editor's live membership/role, authorize, and mutate in one transaction. HookAuthor no longer carries a handler-computed IsWorkspaceAdmin snapshot. 4. decodeHookSpec now rejects a smuggled trailing second JSON document (a second Decode must return io.EOF), which DisallowUnknownFields alone did not catch. Tests: admission unit tests; stored-principal admission (admin cannot add a private agent A can't invoke → 400; owner's own hook may → 201); run_autopilot write-permission (non-writer member → 400, creator → 201); issue_field operand validation; non-member principal rejected at the service (nothing persisted); trailing-JSON rejection. Existing CRUD/auth/concurrency/strict-schema/fail-closed suites still pass. Co-authored-by: multica-agent --- server/internal/admission/admission.go | 77 +++++ server/internal/admission/admission_test.go | 64 ++++ server/internal/handler/agent_access.go | 13 +- server/internal/handler/autopilot.go | 6 +- server/internal/handler/hook.go | 48 ++- server/internal/handler/hook_test.go | 181 ++++++++++- server/internal/service/hook.go | 338 +++++++++++++------- 7 files changed, 565 insertions(+), 162 deletions(-) create mode 100644 server/internal/admission/admission.go create mode 100644 server/internal/admission/admission_test.go diff --git a/server/internal/admission/admission.go b/server/internal/admission/admission.go new file mode 100644 index 00000000000..1009a2eba3c --- /dev/null +++ b/server/internal/admission/admission.go @@ -0,0 +1,77 @@ +// Package admission holds the pure member-permission predicates shared by the +// HTTP handlers and the automation hook service, so both judge agent invocation +// and autopilot write access with identical semantics. The functions here take +// already-loaded rows and make no DB calls, which lets the hook service evaluate +// them inside its write transaction against a hook's stored principal (MUL-4332 +// PR2 review round 3). +package admission + +import ( + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// MemberHitsInvocationTargets reports whether a member is on a public_to agent's +// allow-list: a workspace target admits any member; a member target admits the +// matching user; team targets are inert in v1. +func MemberHitsInvocationTargets(targets []db.AgentInvocationTarget, userID string) bool { + for _, t := range targets { + switch t.TargetType { + case "workspace": + return true + case "member": + if util.UUIDToString(t.TargetID) == userID { + return true + } + } + } + return false +} + +// AgentInvocableByMember mirrors Handler.canInvokeAgent for a member principal: +// the agent owner may always invoke; otherwise a public_to agent is invocable +// only when the member is on its allow-list (a workspace target requires the +// member to still be a current member of the workspace). There is no admin +// bypass — an admin editing a hook may not grant a stored principal reach the +// principal does not have. +func AgentInvocableByMember(agent db.Agent, targets []db.AgentInvocationTarget, memberUserID string, isCurrentMember bool) bool { + if memberUserID != "" && util.UUIDToString(agent.OwnerID) == memberUserID { + return true + } + if agent.PermissionMode != "public_to" { + return false + } + for _, t := range targets { + switch t.TargetType { + case "workspace": + if isCurrentMember { + return true + } + case "member": + if memberUserID != "" && util.UUIDToString(t.TargetID) == memberUserID { + return true + } + } + } + return false +} + +// AutopilotWriteByOwnership reports whether a member may write an autopilot by +// role or authorship (workspace owner/admin, or the member who created it). +// Collaborator grants are checked separately by the caller (a DB lookup). +func AutopilotWriteByOwnership(ap db.Autopilot, member db.Member) bool { + if RoleAllowed(member.Role, "owner", "admin") { + return true + } + return ap.CreatedByType == "member" && util.UUIDToString(ap.CreatedByID) == util.UUIDToString(member.UserID) +} + +// RoleAllowed reports whether role is one of the allowed workspace roles. +func RoleAllowed(role string, allowed ...string) bool { + for _, a := range allowed { + if role == a { + return true + } + } + return false +} diff --git a/server/internal/admission/admission_test.go b/server/internal/admission/admission_test.go new file mode 100644 index 00000000000..8c23492f7cc --- /dev/null +++ b/server/internal/admission/admission_test.go @@ -0,0 +1,64 @@ +package admission + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgtype" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +func uid(t *testing.T, s string) pgtype.UUID { + t.Helper() + var u pgtype.UUID + if err := u.Scan(s); err != nil { + t.Fatalf("bad uuid %q: %v", s, err) + } + return u +} + +const ( + owner = "11111111-1111-1111-1111-111111111111" + member = "22222222-2222-2222-2222-222222222222" +) + +func TestAgentInvocableByMember(t *testing.T) { + privateAgent := db.Agent{OwnerID: uid(t, owner), PermissionMode: "private"} + publicAgent := db.Agent{OwnerID: uid(t, owner), PermissionMode: "public_to"} + wsTarget := []db.AgentInvocationTarget{{TargetType: "workspace"}} + memberTarget := []db.AgentInvocationTarget{{TargetType: "member", TargetID: uid(t, member)}} + + if !AgentInvocableByMember(privateAgent, nil, owner, true) { + t.Error("owner must be able to invoke their own private agent") + } + if AgentInvocableByMember(privateAgent, wsTarget, member, true) { + t.Error("a private agent must not be invocable by a non-owner member") + } + if !AgentInvocableByMember(publicAgent, wsTarget, member, true) { + t.Error("a public_to workspace-target agent is invocable by a current member") + } + if AgentInvocableByMember(publicAgent, wsTarget, member, false) { + t.Error("a workspace target must NOT admit a non-member") + } + if !AgentInvocableByMember(publicAgent, memberTarget, member, true) { + t.Error("a member target admits the matching member") + } + if AgentInvocableByMember(publicAgent, memberTarget, "33333333-3333-3333-3333-333333333333", true) { + t.Error("a member target must not admit a different member") + } +} + +func TestAutopilotWriteByOwnership(t *testing.T) { + byMember := func(role string) db.Member { return db.Member{Role: role, UserID: uid(t, member)} } + ownedByMember := db.Autopilot{CreatedByType: "member", CreatedByID: uid(t, member)} + ownedByOther := db.Autopilot{CreatedByType: "member", CreatedByID: uid(t, owner)} + + if !AutopilotWriteByOwnership(ownedByOther, byMember("admin")) { + t.Error("admin may write any autopilot") + } + if !AutopilotWriteByOwnership(ownedByMember, byMember("member")) { + t.Error("the creating member may write their own autopilot") + } + if AutopilotWriteByOwnership(ownedByOther, byMember("member")) { + t.Error("a plain member may not write another member's autopilot") + } +} diff --git a/server/internal/handler/agent_access.go b/server/internal/handler/agent_access.go index 53968c122ef..8fdd7493f36 100644 --- a/server/internal/handler/agent_access.go +++ b/server/internal/handler/agent_access.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/admission" "github.com/multica-ai/multica/server/internal/util" db "github.com/multica-ai/multica/server/pkg/db/generated" ) @@ -146,17 +147,7 @@ func (h *Handler) canAccessPrivateAgent(ctx context.Context, agent db.Agent, act // view gate and the ListAgents batch filter. A workspace target admits any // member; a member target admits the matching user; team targets are inert. func memberHitsInvocationTargets(targets []db.AgentInvocationTarget, userID string) bool { - for _, t := range targets { - switch t.TargetType { - case "workspace": - return true - case "member": - if uuidToString(t.TargetID) == userID { - return true - } - } - } - return false + return admission.MemberHitsInvocationTargets(targets, userID) } // memberAllowedToViewAgent is the ListAgents / aggregation filter predicate. diff --git a/server/internal/handler/autopilot.go b/server/internal/handler/autopilot.go index a157bcfa043..50af94c62a4 100644 --- a/server/internal/handler/autopilot.go +++ b/server/internal/handler/autopilot.go @@ -13,6 +13,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/admission" "github.com/multica-ai/multica/server/internal/analytics" obsmetrics "github.com/multica-ai/multica/server/internal/metrics" "github.com/multica-ai/multica/server/internal/service" @@ -536,10 +537,7 @@ func (h *Handler) loadAutopilotInWorkspace(w http.ResponseWriter, r *http.Reques // write access. Explicit collaborator grants (memberCanWriteAutopilot) layer // on top of this (MUL-3807). func autopilotWriteByOwnership(ap db.Autopilot, member db.Member) bool { - if roleAllowed(member.Role, "owner", "admin") { - return true - } - return ap.CreatedByType == "member" && uuidToString(ap.CreatedByID) == uuidToString(member.UserID) + return admission.AutopilotWriteByOwnership(ap, member) } // memberCanWriteAutopilot reports whether the given member may perform write or diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go index e52c388ede7..11a8e89070a 100644 --- a/server/internal/handler/hook.go +++ b/server/internal/handler/hook.go @@ -3,6 +3,7 @@ package handler import ( "encoding/json" "errors" + "io" "net/http" "github.com/go-chi/chi/v5" @@ -134,12 +135,12 @@ func (h *Handler) CreateHook(w http.ResponseWriter, r *http.Request) { return } - author, canInvoke, ok := h.resolveHookWriter(w, r, userID, workspaceID) + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } - result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author, canInvoke) + result, err := h.HookService.CreateHook(r.Context(), workspaceUUID, spec, author) if err != nil { h.writeHookError(w, err) return @@ -204,11 +205,11 @@ func (h *Handler) UpdateHook(w http.ResponseWriter, r *http.Request) { if !ok { return } - author, canInvoke, ok := h.resolveHookWriter(w, r, userID, workspaceID) + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } - result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author, canInvoke) + result, err := h.HookService.UpdateHook(r.Context(), workspaceUUID, hookUUID, spec, author) if err != nil { h.writeHookError(w, err) return @@ -250,7 +251,7 @@ func (h *Handler) setHookEnabled(w http.ResponseWriter, r *http.Request, enabled } reason = body.Reason } - author, _, ok := h.resolveHookWriter(w, r, userID, workspaceID) + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } @@ -276,7 +277,7 @@ func (h *Handler) DeleteHook(w http.ResponseWriter, r *http.Request) { if !ok { return } - author, _, ok := h.resolveHookWriter(w, r, userID, workspaceID) + author, ok := h.resolveHookWriter(w, r, userID, workspaceID) if !ok { return } @@ -334,6 +335,13 @@ func decodeHookSpec(w http.ResponseWriter, r *http.Request) (automation.HookSpec writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error()) return automation.HookSpec{}, false } + // The body must be exactly one JSON value. A second Decode must hit io.EOF; + // anything else means trailing data (a smuggled second document) that + // DisallowUnknownFields alone does not catch (MUL-4332 review round 3, point 4). + if err := dec.Decode(new(json.RawMessage)); err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body: expected exactly one JSON document") + return automation.HookSpec{}, false + } return spec, true } @@ -344,39 +352,27 @@ func decodeHookSpec(w http.ResponseWriter, r *http.Request) (automation.HookSpec // behind its current task. The resolved principal must still be a workspace // member (review point 2) — an agent UUID is never a substitute for a human, and // a departed principal can no longer author hooks. -func (h *Handler) resolveHookWriter(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, service.CanInvokeAgent, bool) { +func (h *Handler) resolveHookWriter(w http.ResponseWriter, r *http.Request, userID, workspaceID string) (service.HookAuthor, bool) { actorType, actorID := h.resolveActor(r, userID, workspaceID) actorUUID, err := util.ParseUUID(actorID) if err != nil { writeError(w, http.StatusBadRequest, "invalid actor id") - return service.HookAuthor{}, nil, false + return service.HookAuthor{}, false } principal := h.invokeOriginatorFromRequest(r, actorType, actorID) if principal == "" { h.writeDispatchBlocked(w, http.StatusForbidden, ReasonInvocationNotAllowed) - return service.HookAuthor{}, nil, false + return service.HookAuthor{}, false } principalUUID, err := util.ParseUUID(principal) if err != nil { writeError(w, http.StatusForbidden, "no accountable authorization principal") - return service.HookAuthor{}, nil, false - } - // The accountable principal must still be a member of this workspace. - member, err := h.getWorkspaceMember(r.Context(), principal, workspaceID) - if err != nil { - writeError(w, http.StatusForbidden, "authorization principal is not a member of this workspace") - return service.HookAuthor{}, nil, false - } - author := service.HookAuthor{ - ActorType: actorType, - ActorID: actorUUID, - PrincipalUserID: principalUUID, - IsWorkspaceAdmin: roleAllowed(member.Role, "owner", "admin"), - } - canInvoke := func(agent db.Agent) bool { - return h.canInvokeAgent(r.Context(), agent, actorType, actorID, principal, workspaceID) + return service.HookAuthor{}, false } - return author, canInvoke, true + // Membership, role, and every target admission are (re)checked by the service + // inside the write transaction against this principal, so a stale snapshot can + // never authorize the write. + return service.HookAuthor{ActorType: actorType, ActorID: actorUUID, PrincipalUserID: principalUUID}, true } func (h *Handler) writeHookError(w http.ResponseWriter, err error) { diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go index df68606f36d..bde2de93f68 100644 --- a/server/internal/handler/hook_test.go +++ b/server/internal/handler/hook_test.go @@ -409,10 +409,9 @@ func TestHookConcurrentPatchAppendsContiguousRevisions(t *testing.T) { hook := createHookAs(t, testUserID, sampleHookSpec("concurrent", "hi", issueID)) admin := service.HookAuthor{ - ActorType: "member", - ActorID: parseUUID(testUserID), - PrincipalUserID: parseUUID(testUserID), - IsWorkspaceAdmin: true, + ActorType: "member", + ActorID: parseUUID(testUserID), + PrincipalUserID: parseUUID(testUserID), } hookUUID := parseUUID(hook.ID) wsUUID := parseUUID(testWorkspaceID) @@ -427,7 +426,7 @@ func TestHookConcurrentPatchAppendsContiguousRevisions(t *testing.T) { defer wg.Done() <-start spec := hookSpecFromMap(sampleHookSpec(fmt.Sprintf("rev-%d", n), "x", issueID)) - _, err := testHandler.HookService.UpdateHook(ctx, wsUUID, hookUUID, spec, admin, nil) + _, err := testHandler.HookService.UpdateHook(ctx, wsUUID, hookUUID, spec, admin) errs <- err }(i) } @@ -462,6 +461,178 @@ func TestHookConcurrentPatchAppendsContiguousRevisions(t *testing.T) { } } +// seedPrivateAgent inserts a private agent owned by ownerUserID. +func seedPrivateAgent(t *testing.T, ownerUserID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO agent (workspace_id, name, description, runtime_mode, runtime_config, + runtime_id, visibility, permission_mode, max_concurrent_tasks, owner_id) + VALUES ($1, $2, '', 'cloud', '{}'::jsonb, $3, 'private', 'private', 1, $4) + RETURNING id`, + testWorkspaceID, fmt.Sprintf("Private Agent %d", hookSeedCounter.Add(1)), testRuntimeID, ownerUserID).Scan(&id); err != nil { + t.Fatalf("seed private agent: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM agent WHERE id = $1`, id) }) + return id +} + +// seedAutopilot inserts an autopilot created by creatorUserID. +func seedAutopilot(t *testing.T, creatorUserID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO autopilot (workspace_id, title, assignee_type, assignee_id, status, execution_mode, created_by_type, created_by_id) + VALUES ($1, $2, 'agent', $3, 'active', 'run_only', 'member', $4) + RETURNING id`, + testWorkspaceID, "hook autopilot", seededHookAgentID(t), creatorUserID).Scan(&id); err != nil { + t.Fatalf("seed autopilot: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM autopilot WHERE id = $1`, id) }) + return id +} + +func triggerAgentSpec(name, issueID, agentID string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{"event": "issue.status_changed"}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "trigger_agent", "issue_id": issueID, "agent_id": agentID}}, + } +} + +func runAutopilotSpec(name, autopilotID string) map[string]any { + return map[string]any{ + "name": name, + "when": map[string]any{"event": "issue.status_changed"}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "run_autopilot", "autopilot_id": autopilotID}}, + } +} + +// Target admission is judged against the hook's STORED principal, not the editor. +// An admin editing member A's hook cannot smuggle in a trigger_agent that A could +// not invoke (review round 3, point 1). +func TestHookAdmissionUsesStoredPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + privateAgent := seedPrivateAgent(t, testUserID) // owned by the fixture owner + memberA := seedHookMember(t, "member") + + hook := createHookAs(t, memberA, sampleHookSpec("owned by A", "hi", issueID)) + + // Owner (admin) PATCHes A's hook to trigger the owner's private agent. Because + // admission is judged for A (the stored principal), who cannot invoke that + // private agent, the edit is rejected — the admin cannot expand A's reach. + w := httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, triggerAgentSpec("hijack", issueID, privateAgent)), "id", hook.ID)) + if w.Code != http.StatusBadRequest { + t.Errorf("admin PATCH adding a private agent A cannot invoke: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The owner's OWN hook may reference the owner's private agent (owner invokes it). + createHookAs(t, testUserID, triggerAgentSpec("owner hook", issueID, privateAgent)) +} + +// run_autopilot targets are gated by the stored principal's write permission on +// the autopilot, not mere existence (review round 3, point 2). +func TestHookRunAutopilotChecksWritePermission(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + memberA := seedHookMember(t, "member") + autopilotID := seedAutopilot(t, testUserID) // created by the owner + + // Member A has no write access to the owner's autopilot → rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", runAutopilotSpec("by A", autopilotID), memberA)) + if w.Code != http.StatusBadRequest { + t.Errorf("member A run_autopilot on autopilot they cannot write: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The autopilot's creator (owner) may reference it. + createHookAs(t, testUserID, runAutopilotSpec("by owner", autopilotID)) +} + +// issue_field operand ids must resolve to workspace resources, not just the +// subject issue (review round 3, point 2). +func TestHookIssueFieldOperandValidation(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + realIssue := seedHookIssue(t) + const ghost = "77777777-7777-7777-7777-777777777777" + + parentCond := func(operand string) map[string]any { + return map[string]any{ + "name": "operand", + "when": map[string]any{"event": "issue.status_changed"}, + "if": []any{map[string]any{"issue_field": map[string]any{"id": realIssue, "field": "parent_issue_id", "eq": operand}}}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "add_comment", "issue_id": realIssue, "message": "hi"}}, + } + } + // A ghost parent operand is rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", parentCond(ghost))) + if w.Code != http.StatusBadRequest { + t.Errorf("issue_field parent operand ghost: status %d, want 400: %s", w.Code, w.Body.String()) + } + // A real parent operand is accepted. + createHookAs(t, testUserID, parentCond(realIssue)) +} + +// The write transaction re-checks that the principal is a workspace member; a +// valid-but-non-member principal cannot persist a hook (review round 3, point 3). +func TestHookServiceRejectsNonMemberPrincipal(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + issueID := seedHookIssue(t) + author := service.HookAuthor{ + ActorType: "member", + ActorID: parseUUID("88888888-8888-8888-8888-888888888888"), + PrincipalUserID: parseUUID("88888888-8888-8888-8888-888888888888"), // not a member + } + _, err := testHandler.HookService.CreateHook(ctx, parseUUID(testWorkspaceID), hookSpecFromMap(sampleHookSpec("ghost", "hi", issueID)), author) + if err == nil { + t.Fatalf("expected rejection for non-member principal, got nil") + } + var count int + testPool.QueryRow(ctx, `SELECT count(*) FROM hook WHERE authorization_principal_user_id = $1`, "88888888-8888-8888-8888-888888888888").Scan(&count) + if count != 0 { + t.Errorf("a hook was persisted for a non-member principal (count=%d)", count) + } +} + +// The body must be exactly one JSON document; a smuggled trailing document is +// rejected even though DisallowUnknownFields alone would accept it (review round +// 3, point 4). +func TestHookRejectsTrailingJSONDocument(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + first, _ := json.Marshal(sampleHookSpec("first", "hi", issueID)) + body := append(append([]byte{}, first...), []byte(` {"unexpected":"second document"}`)...) + req := httptest.NewRequest(http.MethodPost, "/api/hooks", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-User-ID", testUserID) + req.Header.Set("X-Workspace-ID", testWorkspaceID) + w := httptest.NewRecorder() + testHandler.CreateHook(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("trailing second JSON document: status %d, want 400: %s", w.Code, w.Body.String()) + } +} + // An agent author with no resolvable human principal (§8) is refused. func TestHookAgentRequiresPrincipal(t *testing.T) { if testPool == nil { diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go index b0fa44fccf9..4980401aee2 100644 --- a/server/internal/service/hook.go +++ b/server/internal/service/hook.go @@ -9,13 +9,14 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/multica-ai/multica/server/internal/admission" "github.com/multica-ai/multica/server/internal/automation" "github.com/multica-ai/multica/server/internal/util" db "github.com/multica-ai/multica/server/pkg/db/generated" ) // Hook CRUD errors surfaced to the handler for status mapping. Validation -// problems (shape or unresolvable target) flow through as +// problems (shape or unresolvable/forbidden target) flow through as // *automation.ValidationError (→ 400). var ( ErrHookNotFound = errors.New("hook not found") @@ -25,22 +26,16 @@ var ( ) // HookAuthor carries the resolved identity for a hook write: who is acting -// (creator, pure audit), the accountable human whose authority the hook runs -// under (§8), and whether that human is a workspace owner/admin. An agent author -// must resolve to a real member principal. +// (creator, pure audit) and the accountable human whose authority the hook runs +// under (§8). Membership and role are NOT carried here — the service re-derives +// them inside the write transaction so a stale snapshot can never authorize a +// write (review round 3, point 3). type HookAuthor struct { - ActorType string // member | agent - ActorID pgtype.UUID - PrincipalUserID pgtype.UUID - IsWorkspaceAdmin bool + ActorType string // member | agent + ActorID pgtype.UUID + PrincipalUserID pgtype.UUID } -// CanInvokeAgent is the admission predicate the handler supplies (wrapping -// Handler.canInvokeAgent) so the service can fail-closed on a trigger_agent -// target without importing request context. A nil predicate denies every -// trigger_agent target (fail-closed). -type CanInvokeAgent func(agent db.Agent) bool - // HookWithRevision pairs a hook row with its active revision so the handler can // render one complete view. The service returns db rows; the handler shapes JSON. type HookWithRevision struct { @@ -62,11 +57,14 @@ func NewHookService(q *db.Queries, tx TxStarter) *HookService { return &HookService{Queries: q, TxStarter: tx} } -// CreateHook validates the spec (shape + workspace-scoped targets), resolves -// scope + principal, and inserts the hook together with revision #1 in one -// transaction. The two rows reference each other, so both ids are generated up -// front. -func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor, canInvoke CanInvokeAgent) (HookWithRevision, error) { +// CreateHook validates the spec (shape + workspace-scoped, principal-gated +// targets) and inserts the hook together with revision #1 in one transaction. +// The writer's membership and every target admission are (re)checked inside that +// transaction against the accountable principal, so an illegal configuration +// never enters the store (§13) and a stale role/membership snapshot cannot +// authorize the write. The two rows reference each other, so both ids are +// generated up front. +func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { if err := automation.Validate(spec); err != nil { return HookWithRevision{}, err } @@ -87,7 +85,17 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { - if err := validateTargets(ctx, qtx, workspaceID, spec, canInvoke); err != nil { + // The creator's principal must be a current member of the workspace. + if _, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + UserID: author.PrincipalUserID, WorkspaceID: workspaceID, + }); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrHookNoPrincipal + } + return err + } + // A newly created hook runs under the creator's authority. + if err := validateTargets(ctx, qtx, workspaceID, spec, util.UUIDToString(author.PrincipalUserID)); err != nil { return err } hook, err := qtx.CreateHook(ctx, db.CreateHookParams{ @@ -131,11 +139,14 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s } // UpdateHook appends a new immutable revision from the spec and repoints the -// hook's active revision (§5.1). It locks the hook row first so concurrent -// PATCHes serialize and MAX(revision)+1 can never collide (review point 4), and -// re-checks archived/origin/authorization inside the lock. Only the hook's -// principal or a workspace admin may edit (review point 1). Scope is immutable. -func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor, canInvoke CanInvokeAgent) (HookWithRevision, error) { +// hook's active revision (§5.1), all inside one transaction that first locks the +// hook row (so concurrent PATCHes serialize and MAX(revision)+1 cannot collide), +// then re-checks archived/origin, the editor's live membership/role, and the +// edit-authorization gate. Target admission is judged against the hook's LOCKED +// stored principal — never the editor — so an admin editing another member's +// hook can only change configuration, never grant the stored principal reach it +// lacks (review round 3, point 1). Scope is immutable. +func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype.UUID, spec automation.HookSpec, author HookAuthor) (HookWithRevision, error) { if err := automation.Validate(spec); err != nil { return HookWithRevision{}, err } @@ -147,23 +158,13 @@ func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype revisionID := util.NewUUID() var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { - existing, err := qtx.GetHookForUpdate(ctx, db.GetHookForUpdateParams{ID: hookID, WorkspaceID: workspaceID}) + existing, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author) if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return ErrHookNotFound - } return err } - if existing.ArchivedAt.Valid { - return ErrHookNotFound - } - if existing.Origin == "system" { - return ErrHookSystemManaged - } - if err := authorizeHookEdit(existing, author); err != nil { - return err - } - if err := validateTargets(ctx, qtx, workspaceID, spec, canInvoke); err != nil { + // Admission uses the hook's STORED principal, resolved from the locked row. + storedPrincipal := util.UUIDToString(existing.AuthorizationPrincipalUserID) + if err := validateTargets(ctx, qtx, workspaceID, spec, storedPrincipal); err != nil { return err } maxRev, err := qtx.GetMaxHookRevision(ctx, hookID) @@ -242,48 +243,49 @@ func (s *HookService) ListHooks(ctx context.Context, workspaceID pgtype.UUID) ([ } // SetEnabled enables/disables a hook. Disable only blocks future matches; it -// does not cancel queued/running executions (§5.1). Only the principal or a -// workspace admin may toggle it. +// does not cancel queued/running executions (§5.1). Load, authorization and the +// mutation all happen inside one transaction against the locked row. func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype.UUID, enabled bool, reason string, author HookAuthor) (HookWithRevision, error) { - if _, err := s.loadEditableHook(ctx, workspaceID, hookID, author); err != nil { - return HookWithRevision{}, err - } disabledReason := pgtype.Text{} if !enabled && reason != "" { disabledReason = pgtype.Text{String: reason, Valid: true} } - hook, err := s.Queries.SetHookEnabled(ctx, db.SetHookEnabledParams{ - ID: hookID, - WorkspaceID: workspaceID, - Enabled: enabled, - DisabledReason: disabledReason, - }) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return HookWithRevision{}, ErrHookNotFound + var out HookWithRevision + err := s.inTx(ctx, func(qtx *db.Queries) error { + if _, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author); err != nil { + return err } - return HookWithRevision{}, err - } - rev, err := s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) - if err != nil { - return HookWithRevision{}, err - } - return HookWithRevision{Hook: hook, Revision: rev}, nil + hook, err := qtx.SetHookEnabled(ctx, db.SetHookEnabledParams{ + ID: hookID, + WorkspaceID: workspaceID, + Enabled: enabled, + DisabledReason: disabledReason, + }) + if err != nil { + return err + } + rev, err := qtx.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return err + } + out = HookWithRevision{Hook: hook, Revision: rev} + return nil + }) + return out, err } // ArchiveHook soft-deletes a hook (§5.1); revisions/executions/effects are kept. -// Only the principal or a workspace admin may archive it. +// Load, authorization and the mutation all happen inside one transaction. func (s *HookService) ArchiveHook(ctx context.Context, workspaceID, hookID pgtype.UUID, author HookAuthor) error { - if _, err := s.loadEditableHook(ctx, workspaceID, hookID, author); err != nil { - return err - } - if _, err := s.Queries.ArchiveHook(ctx, db.ArchiveHookParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return ErrHookNotFound + return s.inTx(ctx, func(qtx *db.Queries) error { + if _, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author); err != nil { + return err } - return err - } - return nil + if _, err := qtx.ArchiveHook(ctx, db.ArchiveHookParams{ID: hookID, WorkspaceID: workspaceID}); err != nil { + return err + } + return nil + }) } // ListExecutions returns the newest execution-trace rows for a hook (bounded). @@ -298,10 +300,12 @@ func (s *HookService) ListExecutions(ctx context.Context, workspaceID, hookID pg return s.Queries.ListHookExecutionsByHook(ctx, db.ListHookExecutionsByHookParams{HookID: hookID, Limit: limit}) } -// loadEditableHook loads a non-archived, non-system hook and enforces the edit -// authorization gate, for the enable/disable/archive paths. -func (s *HookService) loadEditableHook(ctx context.Context, workspaceID, hookID pgtype.UUID, author HookAuthor) (db.Hook, error) { - existing, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) +// lockEditableHook loads and row-locks a non-archived, non-system hook and +// enforces the edit-authorization gate against the editor's LIVE membership and +// role read inside the same transaction. Returns the locked row for callers that +// need its stored principal. +func (s *HookService) lockEditableHook(ctx context.Context, qtx *db.Queries, workspaceID, hookID pgtype.UUID, author HookAuthor) (db.Hook, error) { + existing, err := qtx.GetHookForUpdate(ctx, db.GetHookForUpdateParams{ID: hookID, WorkspaceID: workspaceID}) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return db.Hook{}, ErrHookNotFound @@ -314,22 +318,32 @@ func (s *HookService) loadEditableHook(ctx context.Context, workspaceID, hookID if existing.Origin == "system" { return db.Hook{}, ErrHookSystemManaged } - if err := authorizeHookEdit(existing, author); err != nil { + editor, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + UserID: author.PrincipalUserID, WorkspaceID: workspaceID, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // The editor is not (or no longer) a member of this workspace. + return db.Hook{}, ErrHookForbidden + } + return db.Hook{}, err + } + if err := authorizeHookEdit(existing, editor); err != nil { return db.Hook{}, err } return existing, nil } // authorizeHookEdit implements the edit gate (review point 1): a workspace -// owner/admin may edit any hook; otherwise only the hook's original -// authorization principal may. The principal is NOT transferred on edit, so an -// arbitrary member can never rewrite a rule that keeps running under someone -// else's authority. -func authorizeHookEdit(hook db.Hook, author HookAuthor) error { - if author.IsWorkspaceAdmin { +// owner/admin may edit any hook's configuration; otherwise only the hook's +// original authorization principal may. The principal is NOT transferred on +// edit, so an arbitrary member can never rewrite a rule that keeps running under +// someone else's authority. +func authorizeHookEdit(hook db.Hook, editor db.Member) error { + if admission.RoleAllowed(editor.Role, "owner", "admin") { return nil } - if principalMatches(hook.AuthorizationPrincipalUserID, author.PrincipalUserID) { + if principalMatches(hook.AuthorizationPrincipalUserID, editor.UserID) { return nil } return ErrHookForbidden @@ -383,63 +397,117 @@ func marshalRevisionConfig(spec automation.HookSpec) (match, conditions, actions return match, conditions, actions, nil } -// validateTargets fail-closed checks that every id the spec references exists in -// the hook's workspace under the current principal (review point 2). §13 requires -// this at create/update time so an illegal configuration never enters the store -// and never reaches a worker. Uses the tx-bound queries so the checks share the -// write transaction. -func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, canInvoke CanInvokeAgent) error { +// targetChecker fail-closed validates every id a spec references, against the +// workspace and the hook's stored principal, inside the write transaction +// (review round 3, points 1 & 2). §13 requires this at create/update time so an +// illegal configuration never enters the store and never reaches a worker. +type targetChecker struct { + ctx context.Context + qtx *db.Queries + workspaceID pgtype.UUID + principalUserID string + principalMember db.Member + principalIsMember bool +} + +func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, principalUserID string) error { + tc := &targetChecker{ctx: ctx, qtx: qtx, workspaceID: workspaceID, principalUserID: principalUserID} + // Resolve the principal's membership once; agent workspace-target and + // autopilot admission both need it, and a departed principal fails closed. + if pid, err := util.ParseUUID(principalUserID); err == nil { + if m, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{UserID: pid, WorkspaceID: workspaceID}); err == nil { + tc.principalMember = m + tc.principalIsMember = true + } + } + return tc.validate(spec) +} + +func (tc *targetChecker) validate(spec automation.HookSpec) error { if spec.Scope != nil && spec.Scope.Type == automation.ScopeIssue { - if err := requireIssue(ctx, qtx, workspaceID, spec.Scope.ID, "scope.id"); err != nil { + if err := tc.requireIssue(spec.Scope.ID, "scope.id"); err != nil { return err } } for i, cond := range spec.If { - if cond.IssuesStatus != nil { - for _, id := range cond.IssuesStatus.IDs { - if err := requireIssue(ctx, qtx, workspaceID, id, fmt.Sprintf("if[%d].issues_status.ids", i)); err != nil { - return err - } - } + if err := tc.validateConditionTargets(i, cond); err != nil { + return err + } + } + for i, action := range spec.Do { + if err := tc.validateActionTargets(i, action); err != nil { + return err } - if cond.IssueField != nil { - if err := requireIssue(ctx, qtx, workspaceID, cond.IssueField.ID, fmt.Sprintf("if[%d].issue_field.id", i)); err != nil { + } + return nil +} + +func (tc *targetChecker) validateConditionTargets(i int, c automation.ConditionSpec) error { + if c.IssuesStatus != nil { + for _, id := range c.IssuesStatus.IDs { + if err := tc.requireIssue(id, fmt.Sprintf("if[%d].issues_status.ids", i)); err != nil { return err } } } - for i, action := range spec.Do { - if err := validateActionTargets(ctx, qtx, workspaceID, i, action, canInvoke); err != nil { + if c.IssueField != nil { + if err := tc.requireIssue(c.IssueField.ID, fmt.Sprintf("if[%d].issue_field.id", i)); err != nil { return err } + // The operand ids must also resolve to workspace resources. + operands := collectFieldOperands(*c.IssueField) + where := fmt.Sprintf("if[%d].issue_field", i) + switch c.IssueField.Field { + case automation.IssueFieldParentIssueID: + for _, v := range operands { + if err := tc.requireIssue(v, where); err != nil { + return err + } + } + case automation.IssueFieldAssigneeID: + for _, v := range operands { + if err := tc.requireAssignee(v, where); err != nil { + return err + } + } + } } return nil } -func validateActionTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, i int, a automation.ActionSpec, canInvoke CanInvokeAgent) error { +func (tc *targetChecker) validateActionTargets(i int, a automation.ActionSpec) error { where := fmt.Sprintf("do[%d].%s", i, a.Type) switch a.Type { case automation.ActionSetIssueStatus, automation.ActionAddComment: - return requireIssue(ctx, qtx, workspaceID, a.IssueID, where+".issue_id") + return tc.requireIssue(a.IssueID, where+".issue_id") case automation.ActionTriggerAgent: - if err := requireIssue(ctx, qtx, workspaceID, a.IssueID, where+".issue_id"); err != nil { + if err := tc.requireIssue(a.IssueID, where+".issue_id"); err != nil { return err } - return requireInvokableAgent(ctx, qtx, workspaceID, a.AgentID, where+".agent_id", canInvoke) + return tc.requireInvokableAgent(a.AgentID, where+".agent_id") case automation.ActionSendInbox: - return requireMember(ctx, qtx, workspaceID, a.MemberID, where+".member_id") + return tc.requireMember(a.MemberID, where+".member_id") case automation.ActionRunAutopilot: - return requireAutopilot(ctx, qtx, workspaceID, a.AutopilotID, where+".autopilot_id") + return tc.requireWritableAutopilot(a.AutopilotID, where+".autopilot_id") } return nil } -func requireIssue(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { +func collectFieldOperands(c automation.IssueFieldCond) []string { + out := make([]string, 0, len(c.In)+1) + if c.Eq != "" { + out = append(out, c.Eq) + } + out = append(out, c.In...) + return out +} + +func (tc *targetChecker) requireIssue(id, field string) error { uid, err := util.ParseUUID(id) if err != nil { return automation.NewValidationError("%s must be a uuid", field) } - if _, err := qtx.GetIssueInWorkspace(ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if _, err := tc.qtx.GetIssueInWorkspace(tc.ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err != nil { if errors.Is(err, pgx.ErrNoRows) { return automation.NewValidationError("%s references issue %s which does not exist in this workspace", field, id) } @@ -448,12 +516,12 @@ func requireIssue(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, return nil } -func requireMember(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { +func (tc *targetChecker) requireMember(id, field string) error { uid, err := util.ParseUUID(id) if err != nil { return automation.NewValidationError("%s must be a uuid", field) } - if _, err := qtx.GetMemberInWorkspace(ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if _, err := tc.qtx.GetMemberInWorkspace(tc.ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err != nil { if errors.Is(err, pgx.ErrNoRows) { return automation.NewValidationError("%s references member %s which is not in this workspace", field, id) } @@ -462,29 +530,63 @@ func requireMember(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID return nil } -func requireAutopilot(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string) error { +// requireAssignee accepts either a workspace member (by member row id) or a +// workspace agent — issue assignees are polymorphic. +func (tc *targetChecker) requireAssignee(id, field string) error { uid, err := util.ParseUUID(id) if err != nil { return automation.NewValidationError("%s must be a uuid", field) } - if _, err := qtx.GetAutopilotInWorkspace(ctx, db.GetAutopilotInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}); err != nil { + if _, err := tc.qtx.GetMemberInWorkspace(tc.ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil { + return nil + } + if _, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil { + return nil + } + return automation.NewValidationError("%s references assignee %s which is not a member or agent in this workspace", field, id) +} + +func (tc *targetChecker) requireWritableAutopilot(id, field string) error { + uid, err := util.ParseUUID(id) + if err != nil { + return automation.NewValidationError("%s must be a uuid", field) + } + ap, err := tc.qtx.GetAutopilotInWorkspace(tc.ctx, db.GetAutopilotInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}) + if err != nil { if errors.Is(err, pgx.ErrNoRows) { return automation.NewValidationError("%s references autopilot %s which does not exist in this workspace", field, id) } return err } + // Write permission is judged for the hook's stored principal: role/authorship + // or an explicit collaborator grant — the same rule the interactive autopilot + // write path enforces (review round 3, point 2). + if !tc.principalIsMember { + return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + } + if admission.AutopilotWriteByOwnership(ap, tc.principalMember) { + return nil + } + granted, err := tc.qtx.IsAutopilotCollaborator(tc.ctx, db.IsAutopilotCollaboratorParams{AutopilotID: ap.ID, UserID: tc.principalMember.UserID}) + if err != nil { + return err + } + if !granted { + return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + } return nil } // requireInvokableAgent confirms the agent exists in the workspace, is not -// archived, has a runtime, and is invokable by the current principal — the same -// admission the interactive trigger path enforces, applied fail-closed at save. -func requireInvokableAgent(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, id, field string, canInvoke CanInvokeAgent) error { +// archived, has a runtime, and is invokable by the hook's STORED principal — the +// same admission the interactive trigger path enforces, applied fail-closed at +// save against the principal, never the editor (review round 3, point 1). +func (tc *targetChecker) requireInvokableAgent(id, field string) error { uid, err := util.ParseUUID(id) if err != nil { return automation.NewValidationError("%s must be a uuid", field) } - agent, err := qtx.GetAgentInWorkspace(ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: workspaceID}) + agent, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return automation.NewValidationError("%s references agent %s which does not exist in this workspace", field, id) @@ -494,7 +596,11 @@ func requireInvokableAgent(ctx context.Context, qtx *db.Queries, workspaceID pgt if agent.ArchivedAt.Valid || !agent.RuntimeID.Valid { return automation.NewValidationError("%s references agent %s which is archived or has no runtime", field, id) } - if canInvoke == nil || !canInvoke(agent) { + targets, err := tc.qtx.ListAgentInvocationTargets(tc.ctx, agent.ID) + if err != nil { + return err + } + if !admission.AgentInvocableByMember(agent, targets, tc.principalUserID, tc.principalIsMember) { return automation.NewValidationError("%s references agent %s which the hook's principal may not invoke", field, id) } return nil From 6fc24a1d36d6deb7c651398b176fa9560baf43f8 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Thu, 16 Jul 2026 20:11:53 +0800 Subject: [PATCH 10/15] fix(automation): stored-principal live gate + autopilot assignee admission (MUL-4332 PR2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon's round-4 review on the hook policy layer. 1. Stored principal's live membership is now a HARD gate on re-arming a hook. Update and Enable load the FOR UPDATE-locked hook's stored principal as a current workspace member and fail closed (403) if it departed — an admin can no longer re-point or re-enable a departed principal's hook. Disable/archive stay allowed (safe degrading). validateTargets is now only ever called with a pre-resolved live-member principal, and AgentInvocableByMember returns false for a non-member even in the agent-owner branch. 2. run_autopilot admission now also gates the autopilot's ASSIGNEE invocation, not just write ACL: it resolves the assignee to its executing agent (the agent, or a non-archived squad's leader) and requires the stored principal be able to invoke it; a dangling/archived assignee fails closed. A collaborator with write access whose autopilot targets an agent they cannot invoke is now rejected at save. (Shared checkAgentInvocable with trigger_agent.) 3. issue_field.assignee_id operands are validated with the correct identity types: a MEMBER operand is a USER id (GetMemberByUserAndWorkspace, not a member-row id — a real member's user id was previously 400'd), plus agent and non-archived squad. Issue assignees are polymorphic member|agent|squad. Tests: departed-principal gate (update/enable 403, disable/archive allowed); collaborator-write autopilot with un-invocable assignee rejected, owner accepted; assignee_id member user id / agent / squad accepted, ghost rejected; admission unit test for the non-member owner branch. No migration/sqlc changes. Co-authored-by: multica-agent --- server/internal/admission/admission.go | 18 ++- server/internal/admission/admission_test.go | 3 + server/internal/handler/hook.go | 2 + server/internal/handler/hook_test.go | 122 ++++++++++++++- server/internal/service/hook.go | 157 ++++++++++++++++---- 5 files changed, 260 insertions(+), 42 deletions(-) diff --git a/server/internal/admission/admission.go b/server/internal/admission/admission.go index 1009a2eba3c..1e26faafca1 100644 --- a/server/internal/admission/admission.go +++ b/server/internal/admission/admission.go @@ -28,13 +28,19 @@ func MemberHitsInvocationTargets(targets []db.AgentInvocationTarget, userID stri return false } -// AgentInvocableByMember mirrors Handler.canInvokeAgent for a member principal: -// the agent owner may always invoke; otherwise a public_to agent is invocable -// only when the member is on its allow-list (a workspace target requires the -// member to still be a current member of the workspace). There is no admin -// bypass — an admin editing a hook may not grant a stored principal reach the -// principal does not have. +// AgentInvocableByMember decides whether a member principal may invoke an agent, +// for the hook service's fail-closed save-time admission. A principal who is not +// a CURRENT member of the workspace can invoke nothing — not even an agent they +// own — because a hook must run under a live, accountable member (review round 4, +// point 1); this is stricter than the interactive Handler.canInvokeAgent, whose +// owner branch is membership-independent. Given a current member: the agent owner +// may invoke; otherwise a public_to agent is invocable only when the member is on +// its allow-list. There is no admin bypass — an admin editing a hook may not +// grant a stored principal reach the principal does not have. func AgentInvocableByMember(agent db.Agent, targets []db.AgentInvocationTarget, memberUserID string, isCurrentMember bool) bool { + if !isCurrentMember { + return false + } if memberUserID != "" && util.UUIDToString(agent.OwnerID) == memberUserID { return true } diff --git a/server/internal/admission/admission_test.go b/server/internal/admission/admission_test.go index 8c23492f7cc..b9bb567c019 100644 --- a/server/internal/admission/admission_test.go +++ b/server/internal/admission/admission_test.go @@ -30,6 +30,9 @@ func TestAgentInvocableByMember(t *testing.T) { if !AgentInvocableByMember(privateAgent, nil, owner, true) { t.Error("owner must be able to invoke their own private agent") } + if AgentInvocableByMember(privateAgent, nil, owner, false) { + t.Error("a non-member must invoke nothing, even an agent they own") + } if AgentInvocableByMember(privateAgent, wsTarget, member, true) { t.Error("a private agent must not be invocable by a non-owner member") } diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go index 11a8e89070a..9ff499c5ac1 100644 --- a/server/internal/handler/hook.go +++ b/server/internal/handler/hook.go @@ -387,6 +387,8 @@ func (h *Handler) writeHookError(w http.ResponseWriter, err error) { writeError(w, http.StatusForbidden, err.Error()) case errors.Is(err, service.ErrHookForbidden): writeError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, service.ErrHookPrincipalDeparted): + writeError(w, http.StatusForbidden, err.Error()) case errors.Is(err, service.ErrHookNoPrincipal): writeError(w, http.StatusForbidden, "no accountable authorization principal") default: diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go index bde2de93f68..2c485466203 100644 --- a/server/internal/handler/hook_test.go +++ b/server/internal/handler/hook_test.go @@ -477,21 +477,28 @@ func seedPrivateAgent(t *testing.T, ownerUserID string) string { return id } -// seedAutopilot inserts an autopilot created by creatorUserID. +// seedAutopilot inserts an autopilot created by creatorUserID assigned to the +// seeded (workspace-visible) agent. func seedAutopilot(t *testing.T, creatorUserID string) string { + return seedAutopilotWith(t, creatorUserID, "agent", seededHookAgentID(t)) +} + +// seedAutopilotWith inserts an autopilot with an explicit assignee. +func seedAutopilotWith(t *testing.T, creatorUserID, assigneeType, assigneeID string) string { t.Helper() var id string if err := testPool.QueryRow(context.Background(), ` INSERT INTO autopilot (workspace_id, title, assignee_type, assignee_id, status, execution_mode, created_by_type, created_by_id) - VALUES ($1, $2, 'agent', $3, 'active', 'run_only', 'member', $4) + VALUES ($1, $2, $3, $4, 'active', 'run_only', 'member', $5) RETURNING id`, - testWorkspaceID, "hook autopilot", seededHookAgentID(t), creatorUserID).Scan(&id); err != nil { + testWorkspaceID, "hook autopilot", assigneeType, assigneeID, creatorUserID).Scan(&id); err != nil { t.Fatalf("seed autopilot: %v", err) } t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM autopilot WHERE id = $1`, id) }) return id } + func triggerAgentSpec(name, issueID, agentID string) map[string]any { return map[string]any{ "name": name, @@ -633,6 +640,115 @@ func TestHookRejectsTrailingJSONDocument(t *testing.T) { } } +// A hook's stored principal must remain a live member for the hook to be +// re-armed. After the principal leaves the workspace, update and enable are +// blocked (403), but disable/archive stay allowed for admins as safe degrading +// operations (review round 4, point 1). +func TestHookDepartedPrincipalGate(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + + // A leaves the workspace. + if _, err := testPool.Exec(ctx, `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, testWorkspaceID, memberA); err != nil { + t.Fatalf("remove member A: %v", err) + } + + // Owner (admin) may no longer PATCH or enable A's hook (re-arming). + w := httptest.NewRecorder() + testHandler.UpdateHook(w, withURLParam(newMemberHookRequest(http.MethodPatch, "/api/hooks/"+hook.ID, sampleHookSpec("x", "y", issueID)), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("PATCH after principal departed: status %d, want 403: %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.EnableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/enable", nil), "id", hook.ID)) + if w.Code != http.StatusForbidden { + t.Errorf("enable after principal departed: status %d, want 403: %s", w.Code, w.Body.String()) + } + + // Disable and archive are degrading ops an admin may still perform. + w = httptest.NewRecorder() + testHandler.DisableHook(w, withURLParam(newMemberHookRequest(http.MethodPost, "/api/hooks/"+hook.ID+"/disable", nil), "id", hook.ID)) + if w.Code != http.StatusOK { + t.Errorf("disable after principal departed: status %d, want 200 (degrading): %s", w.Code, w.Body.String()) + } + w = httptest.NewRecorder() + testHandler.DeleteHook(w, withURLParam(newMemberHookRequest(http.MethodDelete, "/api/hooks/"+hook.ID, nil), "id", hook.ID)) + if w.Code != http.StatusNoContent { + t.Errorf("archive after principal departed: status %d, want 204 (degrading): %s", w.Code, w.Body.String()) + } +} + +// run_autopilot admission also gates the autopilot's assignee: a principal with +// write access (collaborator) whose autopilot targets an agent they cannot invoke +// is rejected at save (review round 4, point 2). +func TestHookAutopilotAssigneeAdmission(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + privateAgent := seedPrivateAgent(t, testUserID) // owner's private agent + autopilotID := seedAutopilotWith(t, testUserID, "agent", privateAgent) + memberA := seedHookMember(t, "member") + + // A is granted collaborator write access on the autopilot... + if _, err := testPool.Exec(ctx, ` + INSERT INTO autopilot_collaborator (autopilot_id, user_type, user_id, granted_by) + VALUES ($1, 'member', $2, $3)`, autopilotID, memberA, testUserID); err != nil { + t.Fatalf("grant collaborator: %v", err) + } + + // ...but A cannot invoke the autopilot's private-agent assignee, so save fails. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newUserHookRequest(http.MethodPost, "/api/hooks", runAutopilotSpec("by A collab", autopilotID), memberA)) + if w.Code != http.StatusBadRequest { + t.Errorf("collaborator run_autopilot with un-invocable assignee: status %d, want 400: %s", w.Code, w.Body.String()) + } + + // The owner owns the assignee agent, so their own hook is accepted. + createHookAs(t, testUserID, runAutopilotSpec("by owner", autopilotID)) +} + +// issue_field.assignee_id operands are polymorphic: a member operand is a USER id +// (previously wrongly rejected), plus agent and squad (review round 4, point 3). +func TestHookAssigneeOperandTypes(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + realIssue := seedHookIssue(t) + agentID := seededHookAgentID(t) + squadID := seedSquad(t, agentID, fmt.Sprintf("Hook Squad %d", hookSeedCounter.Add(1)), false) + const ghost = "77777777-7777-7777-7777-777777777777" + + assigneeCond := func(operand string) map[string]any { + return map[string]any{ + "name": "assignee", + "when": map[string]any{"event": "issue.status_changed"}, + "if": []any{map[string]any{"issue_field": map[string]any{"id": realIssue, "field": "assignee_id", "eq": operand}}}, + "fire": map[string]any{"mode": "per_event"}, + "do": []any{map[string]any{"type": "add_comment", "issue_id": realIssue, "message": "hi"}}, + } + } + // member USER id, agent id, squad id are all accepted. + createHookAs(t, testUserID, assigneeCond(testUserID)) // member operand is a user id + createHookAs(t, testUserID, assigneeCond(agentID)) + createHookAs(t, testUserID, assigneeCond(squadID)) + + // A ghost assignee operand is rejected. + w := httptest.NewRecorder() + testHandler.CreateHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks", assigneeCond(ghost))) + if w.Code != http.StatusBadRequest { + t.Errorf("ghost assignee operand: status %d, want 400: %s", w.Code, w.Body.String()) + } +} + // An agent author with no resolvable human principal (§8) is refused. func TestHookAgentRequiresPrincipal(t *testing.T) { if testPool == nil { diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go index 4980401aee2..ed05fd949ba 100644 --- a/server/internal/service/hook.go +++ b/server/internal/service/hook.go @@ -19,10 +19,11 @@ import ( // problems (shape or unresolvable/forbidden target) flow through as // *automation.ValidationError (→ 400). var ( - ErrHookNotFound = errors.New("hook not found") - ErrHookSystemManaged = errors.New("system-managed hooks cannot be modified through this API") - ErrHookNoPrincipal = errors.New("no accountable authorization principal for this hook") - ErrHookForbidden = errors.New("only the hook's principal or a workspace admin may modify it") + ErrHookNotFound = errors.New("hook not found") + ErrHookSystemManaged = errors.New("system-managed hooks cannot be modified through this API") + ErrHookNoPrincipal = errors.New("no accountable authorization principal for this hook") + ErrHookForbidden = errors.New("only the hook's principal or a workspace admin may modify it") + ErrHookPrincipalDeparted = errors.New("the hook's authorization principal is no longer a member of this workspace") ) // HookAuthor carries the resolved identity for a hook write: who is acting @@ -86,16 +87,17 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { // The creator's principal must be a current member of the workspace. - if _, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + creator, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ UserID: author.PrincipalUserID, WorkspaceID: workspaceID, - }); err != nil { + }) + if err != nil { if errors.Is(err, pgx.ErrNoRows) { return ErrHookNoPrincipal } return err } // A newly created hook runs under the creator's authority. - if err := validateTargets(ctx, qtx, workspaceID, spec, util.UUIDToString(author.PrincipalUserID)); err != nil { + if err := validateTargets(ctx, qtx, workspaceID, spec, creator); err != nil { return err } hook, err := qtx.CreateHook(ctx, db.CreateHookParams{ @@ -162,9 +164,15 @@ func (s *HookService) UpdateHook(ctx context.Context, workspaceID, hookID pgtype if err != nil { return err } - // Admission uses the hook's STORED principal, resolved from the locked row. - storedPrincipal := util.UUIDToString(existing.AuthorizationPrincipalUserID) - if err := validateTargets(ctx, qtx, workspaceID, spec, storedPrincipal); err != nil { + // Editing re-arms the hook, so its STORED principal must still be a live + // member — an admin cannot re-point a departed principal's hook at new + // targets (review round 4, point 1). Admission then runs against that + // principal, never the editor. + principal, err := s.requireLivePrincipal(ctx, qtx, workspaceID, existing.AuthorizationPrincipalUserID) + if err != nil { + return err + } + if err := validateTargets(ctx, qtx, workspaceID, spec, principal); err != nil { return err } maxRev, err := qtx.GetMaxHookRevision(ctx, hookID) @@ -252,9 +260,18 @@ func (s *HookService) SetEnabled(ctx context.Context, workspaceID, hookID pgtype } var out HookWithRevision err := s.inTx(ctx, func(qtx *db.Queries) error { - if _, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author); err != nil { + existing, err := s.lockEditableHook(ctx, qtx, workspaceID, hookID, author) + if err != nil { return err } + // Enable re-arms the hook, so (like update) its stored principal must still + // be a live member. Disable is a degrading op: an admin may safely disable a + // departed principal's hook (review round 4, point 1). + if enabled { + if _, err := s.requireLivePrincipal(ctx, qtx, workspaceID, existing.AuthorizationPrincipalUserID); err != nil { + return err + } + } hook, err := qtx.SetHookEnabled(ctx, db.SetHookEnabledParams{ ID: hookID, WorkspaceID: workspaceID, @@ -334,6 +351,22 @@ func (s *HookService) lockEditableHook(ctx context.Context, qtx *db.Queries, wor return existing, nil } +// requireLivePrincipal loads the hook's stored principal as a workspace member, +// making live membership a hard precondition for re-arming a hook (update/enable). +// A departed principal fails closed; a real DB error is propagated. +func (s *HookService) requireLivePrincipal(ctx context.Context, qtx *db.Queries, workspaceID, principal pgtype.UUID) (db.Member, error) { + member, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + UserID: principal, WorkspaceID: workspaceID, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return db.Member{}, ErrHookPrincipalDeparted + } + return db.Member{}, err + } + return member, nil +} + // authorizeHookEdit implements the edit gate (review point 1): a workspace // owner/admin may edit any hook's configuration; otherwise only the hook's // original authorization principal may. The principal is NOT transferred on @@ -410,15 +443,17 @@ type targetChecker struct { principalIsMember bool } -func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, principalUserID string) error { - tc := &targetChecker{ctx: ctx, qtx: qtx, workspaceID: workspaceID, principalUserID: principalUserID} - // Resolve the principal's membership once; agent workspace-target and - // autopilot admission both need it, and a departed principal fails closed. - if pid, err := util.ParseUUID(principalUserID); err == nil { - if m, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{UserID: pid, WorkspaceID: workspaceID}); err == nil { - tc.principalMember = m - tc.principalIsMember = true - } +// validateTargets is always called with the hook's stored principal already +// resolved to a LIVE workspace member (the create/update paths make that a hard +// gate first), so every target admission runs against a known-member principal. +func validateTargets(ctx context.Context, qtx *db.Queries, workspaceID pgtype.UUID, spec automation.HookSpec, principal db.Member) error { + tc := &targetChecker{ + ctx: ctx, + qtx: qtx, + workspaceID: workspaceID, + principalUserID: util.UUIDToString(principal.UserID), + principalMember: principal, + principalIsMember: true, } return tc.validate(spec) } @@ -530,20 +565,26 @@ func (tc *targetChecker) requireMember(id, field string) error { return nil } -// requireAssignee accepts either a workspace member (by member row id) or a -// workspace agent — issue assignees are polymorphic. +// requireAssignee validates an issue-assignee operand. Issue assignees are +// polymorphic (member | agent | squad, §Domain): a MEMBER assignee id is a USER +// id (looked up via GetMemberByUserAndWorkspace, NOT a member-row id), an agent +// id is a workspace agent, and a squad id is a non-archived workspace squad +// (review round 4, point 3). func (tc *targetChecker) requireAssignee(id, field string) error { uid, err := util.ParseUUID(id) if err != nil { return automation.NewValidationError("%s must be a uuid", field) } - if _, err := tc.qtx.GetMemberInWorkspace(tc.ctx, db.GetMemberInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil { + if _, err := tc.qtx.GetMemberByUserAndWorkspace(tc.ctx, db.GetMemberByUserAndWorkspaceParams{UserID: uid, WorkspaceID: tc.workspaceID}); err == nil { return nil } if _, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil { return nil } - return automation.NewValidationError("%s references assignee %s which is not a member or agent in this workspace", field, id) + if squad, err := tc.qtx.GetSquadInWorkspace(tc.ctx, db.GetSquadInWorkspaceParams{ID: uid, WorkspaceID: tc.workspaceID}); err == nil && !squad.ArchivedAt.Valid { + return nil + } + return automation.NewValidationError("%s references assignee %s which is not a member, agent, or squad in this workspace", field, id) } func (tc *targetChecker) requireWritableAutopilot(id, field string) error { @@ -564,17 +605,59 @@ func (tc *targetChecker) requireWritableAutopilot(id, field string) error { if !tc.principalIsMember { return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) } - if admission.AutopilotWriteByOwnership(ap, tc.principalMember) { - return nil - } - granted, err := tc.qtx.IsAutopilotCollaborator(tc.ctx, db.IsAutopilotCollaboratorParams{AutopilotID: ap.ID, UserID: tc.principalMember.UserID}) - if err != nil { - return err + if !admission.AutopilotWriteByOwnership(ap, tc.principalMember) { + granted, err := tc.qtx.IsAutopilotCollaborator(tc.ctx, db.IsAutopilotCollaboratorParams{AutopilotID: ap.ID, UserID: tc.principalMember.UserID}) + if err != nil { + return err + } + if !granted { + return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + } } - if !granted { - return automation.NewValidationError("%s references autopilot %s which the hook's principal may not write", field, id) + // Write permission is not enough: running the autopilot invokes its assignee + // (an agent, or a squad's leader agent), so the principal must also be able to + // invoke that agent — otherwise execution would be admission-skipped (review + // round 4, point 2). PR4 re-validates dynamic state at execution time. + return tc.requireInvocableAutopilotAssignee(ap, field) +} + +// requireInvocableAutopilotAssignee resolves the autopilot's assignee to its +// executing agent (the agent itself, or a non-archived squad's leader) and +// requires the stored principal be able to invoke it. A dangling/archived +// assignee fails closed. +func (tc *targetChecker) requireInvocableAutopilotAssignee(ap db.Autopilot, field string) error { + switch ap.AssigneeType { + case "agent": + agent, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: ap.AssigneeID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose assignee agent is missing or not in this workspace", field, util.UUIDToString(ap.ID)) + } + return err + } + return tc.checkAgentInvocable(agent, field) + case "squad": + squad, err := tc.qtx.GetSquadInWorkspace(tc.ctx, db.GetSquadInWorkspaceParams{ID: ap.AssigneeID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose squad is missing or not in this workspace", field, util.UUIDToString(ap.ID)) + } + return err + } + if squad.ArchivedAt.Valid { + return automation.NewValidationError("%s references autopilot %s whose squad is archived", field, util.UUIDToString(ap.ID)) + } + leader, err := tc.qtx.GetAgentInWorkspace(tc.ctx, db.GetAgentInWorkspaceParams{ID: squad.LeaderID, WorkspaceID: tc.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.NewValidationError("%s references autopilot %s whose squad leader is missing", field, util.UUIDToString(ap.ID)) + } + return err + } + return tc.checkAgentInvocable(leader, field) + default: + return automation.NewValidationError("%s references autopilot %s with an unsupported assignee", field, util.UUIDToString(ap.ID)) } - return nil } // requireInvokableAgent confirms the agent exists in the workspace, is not @@ -593,6 +676,14 @@ func (tc *targetChecker) requireInvokableAgent(id, field string) error { } return err } + return tc.checkAgentInvocable(agent, field) +} + +// checkAgentInvocable fail-closed asserts an already-loaded workspace agent is +// live (not archived, has a runtime) and invocable by the hook's stored +// principal. Shared by trigger_agent and run_autopilot assignee validation. +func (tc *targetChecker) checkAgentInvocable(agent db.Agent, field string) error { + id := util.UUIDToString(agent.ID) if agent.ArchivedAt.Valid || !agent.RuntimeID.Valid { return automation.NewValidationError("%s references agent %s which is archived or has no runtime", field, id) } From 309420e29a43033722abd6e1c7b46fd7f1b68ee4 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Fri, 17 Jul 2026 11:48:20 +0800 Subject: [PATCH 11/15] fix(automation): serialize in-tx membership with FOR SHARE (MUL-4332 PR2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon's round-5 must-fix: the hook write transaction's membership checks (create's creator, lockEditableHook's editor, requireLivePrincipal's stored principal) used a plain GetMemberByUserAndWorkspace. Under READ COMMITTED, putting the read inside the transaction does not stop a concurrent member removal or role demotion from committing before the hook write does, so a hook could commit a new revision (or re-enable) under an authority that was revoked mid-transaction. New GetMemberByUserAndWorkspaceForShare (SELECT ... FOR SHARE) takes a shared row lock that blocks a concurrent role UPDATE (demotion) and member DELETE (removal) from committing until the hook transaction ends; conversely, if the removal/ demotion committed first, the FOR SHARE read observes it and fails closed. All three in-transaction membership reads now use it. Concurrent hook writes by the same principal still share-lock without blocking each other. Tests (deterministic, uncommitted mutation held while the op blocks on FOR SHARE, then committed): concurrent stored-principal removal fails Update and Enable with no revision/enable committed; concurrent admin→member editor demotion fails Update. Also cleans up the autopilot_collaborator row TestHookAutopilotAssignee- Admission left behind (no FK/cascade). No migration change; only a new locking query. Co-authored-by: multica-agent --- server/internal/handler/hook_test.go | 153 +++++++++++++++++++++++++- server/internal/service/hook.go | 14 ++- server/pkg/db/generated/member.sql.go | 32 ++++++ server/pkg/db/queries/member.sql | 13 +++ 4 files changed, 207 insertions(+), 5 deletions(-) diff --git a/server/internal/handler/hook_test.go b/server/internal/handler/hook_test.go index 2c485466203..78720e41d1a 100644 --- a/server/internal/handler/hook_test.go +++ b/server/internal/handler/hook_test.go @@ -10,6 +10,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/multica-ai/multica/server/internal/automation" "github.com/multica-ai/multica/server/internal/featureflags" @@ -498,7 +499,6 @@ func seedAutopilotWith(t *testing.T, creatorUserID, assigneeType, assigneeID str return id } - func triggerAgentSpec(name, issueID, agentID string) map[string]any { return map[string]any{ "name": name, @@ -703,6 +703,11 @@ func TestHookAutopilotAssigneeAdmission(t *testing.T) { VALUES ($1, 'member', $2, $3)`, autopilotID, memberA, testUserID); err != nil { t.Fatalf("grant collaborator: %v", err) } + // autopilot_collaborator has no FK/cascade, so drop the grant explicitly to keep + // the shared test DB clean across repeat runs. + t.Cleanup(func() { + testPool.Exec(context.Background(), `DELETE FROM autopilot_collaborator WHERE autopilot_id = $1`, autopilotID) + }) // ...but A cannot invoke the autopilot's private-agent assignee, so save fails. w := httptest.NewRecorder() @@ -749,6 +754,152 @@ func TestHookAssigneeOperandTypes(t *testing.T) { } } +// runBlockedByMemberMutation opens an uncommitted member-row mutation (removal or +// role demotion), runs a hook op in a goroutine that is expected to BLOCK on the +// FOR SHARE membership read, asserts it blocks, then commits the mutation and +// returns the op's error. This deterministically reproduces the round-5 concurrency +// gap: the mutation lands before the hook write can proceed. +func runBlockedByMemberMutation(t *testing.T, mutationSQL string, args []any, op func() error) error { + t.Helper() + ctx := context.Background() + conn, err := testPool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire conn: %v", err) + } + defer conn.Release() + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatalf("begin mutation tx: %v", err) + } + committed := false + defer func() { + if !committed { + tx.Rollback(ctx) + } + }() + if _, err := tx.Exec(ctx, mutationSQL, args...); err != nil { + t.Fatalf("member mutation: %v", err) + } + + done := make(chan error, 1) + go func() { done <- op() }() + + // The op must block on the locked member row while the mutation is uncommitted. + select { + case err := <-done: + t.Fatalf("hook op completed (err=%v) before the concurrent member mutation committed — FOR SHARE did not serialize", err) + case <-time.After(750 * time.Millisecond): + } + + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit member mutation: %v", err) + } + committed = true + return <-done +} + +func hookRevisionCount(t *testing.T, hookID string) int { + t.Helper() + var n int + if err := testPool.QueryRow(context.Background(), `SELECT count(*) FROM hook_revision WHERE hook_id = $1`, hookID).Scan(&n); err != nil { + t.Fatalf("count revisions: %v", err) + } + return n +} + +func memberAuthor(userID string) service.HookAuthor { + return service.HookAuthor{ActorType: "member", ActorID: parseUUID(userID), PrincipalUserID: parseUUID(userID)} +} + +// A concurrent removal of the hook's stored principal, committed while an update is +// mid-transaction, must make the update fail closed — not commit a new revision +// under the departed principal's authority (review round 5). +func TestHookUpdateBlocksOnConcurrentPrincipalRemoval(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + spec := hookSpecFromMap(sampleHookSpec("edited", "x", issueID)) + + err := runBlockedByMemberMutation(t, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, memberA}, + func() error { + // Owner (admin) edits A's hook; admission runs against the stored + // principal A, whose membership is being removed concurrently. + _, e := testHandler.HookService.UpdateHook(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), spec, memberAuthor(testUserID)) + return e + }) + if err == nil { + t.Fatalf("UpdateHook committed after the stored principal was removed mid-transaction") + } + if n := hookRevisionCount(t, hook.ID); n != 1 { + t.Errorf("hook_revision count = %d, want 1 (no revision may commit under a removed principal)", n) + } +} + +// Enable is a re-arming op, so a concurrent removal of the stored principal must +// also block it fail-closed. +func TestHookEnableBlocksOnConcurrentPrincipalRemoval(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + + // Disable it first (allowed) so a failed re-enable is observable. + if _, err := testHandler.HookService.SetEnabled(ctx, parseUUID(testWorkspaceID), parseUUID(hook.ID), false, "", memberAuthor(testUserID)); err != nil { + t.Fatalf("disable: %v", err) + } + + err := runBlockedByMemberMutation(t, + `DELETE FROM member WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, memberA}, + func() error { + _, e := testHandler.HookService.SetEnabled(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), true, "", memberAuthor(testUserID)) + return e + }) + if err == nil { + t.Fatalf("EnableHook committed after the stored principal was removed mid-transaction") + } + var enabled bool + testPool.QueryRow(ctx, `SELECT enabled FROM hook WHERE id = $1`, hook.ID).Scan(&enabled) + if enabled { + t.Errorf("hook was re-enabled despite the principal being removed") + } +} + +// A concurrent admin→member demotion of the EDITOR, committed mid-transaction, must +// make the update fail closed (the editor is no longer authorized). +func TestHookUpdateBlocksOnConcurrentEditorDemotion(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + memberA := seedHookMember(t, "member") // stored principal (stays a member) + adminB := seedHookMember(t, "admin") // editor, demoted mid-transaction + hook := createHookAs(t, memberA, sampleHookSpec("A hook", "hi", issueID)) + spec := hookSpecFromMap(sampleHookSpec("edited", "x", issueID)) + + err := runBlockedByMemberMutation(t, + `UPDATE member SET role = 'member' WHERE workspace_id = $1 AND user_id = $2`, []any{testWorkspaceID, adminB}, + func() error { + _, e := testHandler.HookService.UpdateHook(context.Background(), parseUUID(testWorkspaceID), parseUUID(hook.ID), spec, memberAuthor(adminB)) + return e + }) + if err == nil { + t.Fatalf("UpdateHook committed after the editor was demoted from admin mid-transaction") + } + if n := hookRevisionCount(t, hook.ID); n != 1 { + t.Errorf("hook_revision count = %d, want 1 (no revision may commit under a demoted editor)", n) + } +} + // An agent author with no resolvable human principal (§8) is refused. func TestHookAgentRequiresPrincipal(t *testing.T) { if testPool == nil { diff --git a/server/internal/service/hook.go b/server/internal/service/hook.go index ed05fd949ba..88b8bdbd7d3 100644 --- a/server/internal/service/hook.go +++ b/server/internal/service/hook.go @@ -86,8 +86,10 @@ func (s *HookService) CreateHook(ctx context.Context, workspaceID pgtype.UUID, s var out HookWithRevision err = s.inTx(ctx, func(qtx *db.Queries) error { - // The creator's principal must be a current member of the workspace. - creator, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + // The creator's principal must be a current member of the workspace. The + // FOR SHARE lock blocks a concurrent removal/demotion from committing before + // this hook write does (review round 5). + creator, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ UserID: author.PrincipalUserID, WorkspaceID: workspaceID, }) if err != nil { @@ -335,7 +337,9 @@ func (s *HookService) lockEditableHook(ctx context.Context, qtx *db.Queries, wor if existing.Origin == "system" { return db.Hook{}, ErrHookSystemManaged } - editor, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + // FOR SHARE so a concurrent removal/demotion of the editor cannot commit before + // this edit does — the authorization decision stays consistent to commit. + editor, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ UserID: author.PrincipalUserID, WorkspaceID: workspaceID, }) if err != nil { @@ -355,7 +359,9 @@ func (s *HookService) lockEditableHook(ctx context.Context, qtx *db.Queries, wor // making live membership a hard precondition for re-arming a hook (update/enable). // A departed principal fails closed; a real DB error is propagated. func (s *HookService) requireLivePrincipal(ctx context.Context, qtx *db.Queries, workspaceID, principal pgtype.UUID) (db.Member, error) { - member, err := qtx.GetMemberByUserAndWorkspace(ctx, db.GetMemberByUserAndWorkspaceParams{ + // FOR SHARE so a concurrent removal of the stored principal cannot commit before + // the hook is re-armed under that principal's (now-stale) authority. + member, err := qtx.GetMemberByUserAndWorkspaceForShare(ctx, db.GetMemberByUserAndWorkspaceForShareParams{ UserID: principal, WorkspaceID: workspaceID, }) if err != nil { diff --git a/server/pkg/db/generated/member.sql.go b/server/pkg/db/generated/member.sql.go index 322a22b0cfd..5dfb28ac511 100644 --- a/server/pkg/db/generated/member.sql.go +++ b/server/pkg/db/generated/member.sql.go @@ -86,6 +86,38 @@ func (q *Queries) GetMemberByUserAndWorkspace(ctx context.Context, arg GetMember return i, err } +const getMemberByUserAndWorkspaceForShare = `-- name: GetMemberByUserAndWorkspaceForShare :one +SELECT id, workspace_id, user_id, role, created_at FROM member +WHERE user_id = $1 AND workspace_id = $2 +FOR SHARE +` + +type GetMemberByUserAndWorkspaceForShareParams struct { + UserID pgtype.UUID `json:"user_id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` +} + +// Locking membership read for authorization inside a write transaction. FOR SHARE +// takes a shared row lock that blocks a concurrent role UPDATE (demotion) or member +// DELETE (removal) from committing until this transaction ends, so a hook write can +// never commit under a membership/role that was revoked mid-transaction — a plain +// read under READ COMMITTED would miss that (MUL-4332 PR2 review round 5). Multiple +// concurrent hook writes may still share-lock the same member without blocking each +// other. Use ONLY inside the hook write transaction; the plain variant remains for +// non-transactional reads. +func (q *Queries) GetMemberByUserAndWorkspaceForShare(ctx context.Context, arg GetMemberByUserAndWorkspaceForShareParams) (Member, error) { + row := q.db.QueryRow(ctx, getMemberByUserAndWorkspaceForShare, arg.UserID, arg.WorkspaceID) + var i Member + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.UserID, + &i.Role, + &i.CreatedAt, + ) + return i, err +} + const getMemberInWorkspace = `-- name: GetMemberInWorkspace :one SELECT id, workspace_id, user_id, role, created_at FROM member WHERE id = $1 AND workspace_id = $2 diff --git a/server/pkg/db/queries/member.sql b/server/pkg/db/queries/member.sql index 2c4e471eef9..9ae6ee1b8cf 100644 --- a/server/pkg/db/queries/member.sql +++ b/server/pkg/db/queries/member.sql @@ -18,6 +18,19 @@ WHERE id = $1 AND workspace_id = $2; SELECT * FROM member WHERE user_id = $1 AND workspace_id = $2; +-- name: GetMemberByUserAndWorkspaceForShare :one +-- Locking membership read for authorization inside a write transaction. FOR SHARE +-- takes a shared row lock that blocks a concurrent role UPDATE (demotion) or member +-- DELETE (removal) from committing until this transaction ends, so a hook write can +-- never commit under a membership/role that was revoked mid-transaction — a plain +-- read under READ COMMITTED would miss that (MUL-4332 PR2 review round 5). Multiple +-- concurrent hook writes may still share-lock the same member without blocking each +-- other. Use ONLY inside the hook write transaction; the plain variant remains for +-- non-transactional reads. +SELECT * FROM member +WHERE user_id = $1 AND workspace_id = $2 +FOR SHARE; + -- name: CreateMember :one INSERT INTO member (workspace_id, user_id, role) VALUES ($1, $2, $3) From 32b774e21ec57aa23ae22cdae3962d35e09ed4db Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Fri, 17 Jul 2026 16:46:39 +0800 Subject: [PATCH 12/15] feat(automation): read-only evaluator + dry-run/explain/correlation (MUL-4332 PR3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First PR3 slice, per ratified decision 2A: the single read-only evaluator every later matcher/executor stage will reuse, and the read-only debug surface that depends on it. No execution and no durable-state mutation; the whole surface stays behind the default-off automation_event_hooks flag. - internal/automation/eval.go: the shared evaluator. Given an event + a hook revision + a StateReader, it computes whether the event matches (envelope + payload clauses, event-type gate) and whether the revision's conditions hold against CURRENT workspace state, producing a structured Evaluation with stable reason codes (matched / event_type_mismatch / no_match / condition_false) and evaluated_against=current_state. rising_edge carries an honest read-only note that the durable latch is not consulted. Pure; unit-tested with a fake StateReader. - internal/service/hook_eval.go: DryRun (validate a candidate spec's shape, then evaluate against a historical event), Explain (evaluate a stored hook's active or a specified revision), EventsByCorrelation, and the workspace-scoped issueStateReader. Historical event `when` uses its stored payload; `if` reads current state. Adds GetHookRevisionByNumber. - REST: POST /api/hooks/dry-run, POST /api/hooks/explain, GET /api/events?correlation_id= (flag-gated, strict single-JSON decode). CLI: multica hook dry-run / explain. Docs: issue §14/§15 and the PR description record ratified 1A (principal fixed at creation; adopt/clone deferred) and 2A (dry-run/explain/correlation in PR3, one evaluator), and move dry-run/explain out of PR2's scope. Tests: evaluator units (match ops, all/any conditions, missing issue, issue_field, rising-edge note); handler dry-run (match/no-match/404/invalid), current-state condition (issue todo→done flips the same dry-run), explain (match/no-match/404), correlation isolation, and flag-off 404. No migration change; one new read query. Co-authored-by: multica-agent --- server/cmd/multica/cmd_hook.go | 66 ++++++ server/cmd/server/router.go | 9 + server/internal/automation/eval.go | 272 ++++++++++++++++++++++ server/internal/automation/eval_test.go | 147 ++++++++++++ server/internal/handler/hook.go | 2 + server/internal/handler/hook_eval.go | 161 +++++++++++++ server/internal/handler/hook_eval_test.go | 221 ++++++++++++++++++ server/internal/service/hook_eval.go | 162 +++++++++++++ server/pkg/db/generated/hook.sql.go | 30 +++ server/pkg/db/queries/hook.sql | 5 + 10 files changed, 1075 insertions(+) create mode 100644 server/internal/automation/eval.go create mode 100644 server/internal/automation/eval_test.go create mode 100644 server/internal/handler/hook_eval.go create mode 100644 server/internal/handler/hook_eval_test.go create mode 100644 server/internal/service/hook_eval.go diff --git a/server/cmd/multica/cmd_hook.go b/server/cmd/multica/cmd_hook.go index 135dc397c55..a315c3961ac 100644 --- a/server/cmd/multica/cmd_hook.go +++ b/server/cmd/multica/cmd_hook.go @@ -70,6 +70,19 @@ var hookExecutionsCmd = &cobra.Command{ RunE: runHookExecutions, } +var hookDryRunCmd = &cobra.Command{ + Use: "dry-run", + Short: "Evaluate a candidate hook spec against a historical event (read-only)", + RunE: runHookDryRun, +} + +var hookExplainCmd = &cobra.Command{ + Use: "explain ", + Short: "Explain whether a stored hook would fire for a historical event (read-only)", + Args: exactArgs(1), + RunE: runHookExplain, +} + func init() { hookCmd.AddCommand(hookListCmd) hookCmd.AddCommand(hookGetCmd) @@ -79,6 +92,8 @@ func init() { hookCmd.AddCommand(hookDisableCmd) hookCmd.AddCommand(hookDeleteCmd) hookCmd.AddCommand(hookExecutionsCmd) + hookCmd.AddCommand(hookDryRunCmd) + hookCmd.AddCommand(hookExplainCmd) hookListCmd.Flags().String("output", "table", "Output format: table or json") hookGetCmd.Flags().String("output", "json", "Output format: table or json") @@ -90,6 +105,15 @@ func init() { _ = hookUpdateCmd.MarkFlagRequired("file") hookDisableCmd.Flags().String("reason", "", "Optional reason recorded on the hook") + + hookDryRunCmd.Flags().String("file", "", "Path to a JSON hook spec file (required)") + _ = hookDryRunCmd.MarkFlagRequired("file") + hookDryRunCmd.Flags().String("event", "", "Historical event id to evaluate against (required)") + _ = hookDryRunCmd.MarkFlagRequired("event") + + hookExplainCmd.Flags().String("event", "", "Historical event id to evaluate against (required)") + _ = hookExplainCmd.MarkFlagRequired("event") + hookExplainCmd.Flags().Int("revision", 0, "Explain a specific revision number (default: active revision)") } // readHookSpecFile loads and validates that the spec file is well-formed JSON, @@ -269,6 +293,48 @@ func runHookExecutions(cmd *cobra.Command, args []string) error { return nil } +func runHookDryRun(cmd *cobra.Command, _ []string) error { + path, _ := cmd.Flags().GetString("file") + event, _ := cmd.Flags().GetString("event") + spec, err := readHookSpecFile(path) + if err != nil { + return err + } + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/dry-run", map[string]any{"hook": spec, "event_id": event}, &result); err != nil { + return fmt.Errorf("dry-run hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + +func runHookExplain(cmd *cobra.Command, args []string) error { + event, _ := cmd.Flags().GetString("event") + revision, _ := cmd.Flags().GetInt("revision") + client, err := newAPIClient(cmd) + if err != nil { + return err + } + ctx, cancel := cli.APIContext(context.Background()) + defer cancel() + + body := map[string]any{"hook_id": args[0], "event_id": event} + if revision > 0 { + body["revision"] = revision + } + var result map[string]any + if err := client.PostJSON(ctx, "/api/hooks/explain", body, &result); err != nil { + return fmt.Errorf("explain hook: %w", err) + } + return cli.PrintJSON(os.Stdout, result) +} + // hookRevisionField reads a field out of the nested "revision" object for the // list table view. func hookRevisionField(hook map[string]any, key string) string { diff --git a/server/cmd/server/router.go b/server/cmd/server/router.go index d8c3c209096..e5127b4c5c3 100644 --- a/server/cmd/server/router.go +++ b/server/cmd/server/router.go @@ -1254,6 +1254,11 @@ func NewRouterWithOptions(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus r.Route("/api/hooks", func(r chi.Router) { r.Get("/", h.ListHooks) r.Post("/", h.CreateHook) + // Read-only debug surface (PR3, decision 2A): evaluate a candidate + // spec or a stored hook against a historical event without executing + // anything. chi matches these static routes before the /{id} param. + r.Post("/dry-run", h.DryRunHook) + r.Post("/explain", h.ExplainHook) r.Route("/{id}", func(r chi.Router) { r.Get("/", h.GetHook) r.Patch("/", h.UpdateHook) @@ -1264,6 +1269,10 @@ func NewRouterWithOptions(pool *pgxpool.Pool, hub *realtime.Hub, bus *events.Bus }) }) + // Event Hooks correlation-chain read (PR3, decision 2A), gated on the + // same automation_event_hooks flag inside the handler. + r.Get("/api/events", h.ListEventsByCorrelation) + // Dashboard — workspace-wide token + run-time rollups for the // "/{slug}/dashboard" page. Optional ?project_id filter scopes // the rollup to a single project. diff --git a/server/internal/automation/eval.go b/server/internal/automation/eval.go new file mode 100644 index 00000000000..b013f662f39 --- /dev/null +++ b/server/internal/automation/eval.go @@ -0,0 +1,272 @@ +package automation + +import ( + "context" + "encoding/json" + "fmt" + "sort" +) + +// This file is the single read-only evaluator for Event Hooks (MUL-4332 PR3, +// ratified decision 2A). The matcher (a later PR3 slice) and the dry-run/explain +// debug surface share it, so an explanation can never drift from real execution. +// It computes only WHETHER an event matches a revision and whether the revision's +// conditions hold against current workspace state; it performs NO action and +// mutates NO durable state (no rising-edge latch, rate bucket, execution or +// effect). The `when` match reads the event's own (historical) payload; the `if` +// conditions read current state via StateReader. + +// Stable reason codes. explain/dry-run and (later) the matcher's skip_reason draw +// from this fixed vocabulary so callers can branch on a stable string. +const ( + ReasonMatched = "matched" // matches and conditions currently hold + ReasonEventTypeMismatch = "event_type_mismatch" // the event is not the hook's event type + ReasonNoMatch = "no_match" // a when-clause did not match + ReasonConditionFalse = "condition_false" // matched, but an if-condition is not satisfied +) + +// EvaluatedAgainstCurrentState labels the state basis of a condition evaluation +// (2A): conditions read current workspace state, not the event's point in time. +const EvaluatedAgainstCurrentState = "current_state" + +// StateReader reads the current value of an issue field for condition evaluation. +// Implemented by the service against workspace-scoped queries; kept as an +// interface so this package stays pure and unit-testable. +type StateReader interface { + // IssueField returns the current value of field (status | assignee_id | + // parent_issue_id) for the issue. exists is false when the issue is absent + // from the workspace, in which case the predicate is treated as unsatisfied. + IssueField(ctx context.Context, issueID, field string) (value string, exists bool, err error) +} + +// EventView is the read-only projection of a domain event the evaluator matches +// against: envelope fields plus the decoded payload object. +type EventView struct { + Type string + SubjectID string + ActorType string + ActorID string + Payload map[string]any +} + +// EvalRevision is the parsed hook revision configuration the evaluator needs. +type EvalRevision struct { + EventType string + Match json.RawMessage + Conditions []ConditionSpec + FireMode string +} + +// ClauseResult is the per-field outcome of one when-match clause. +type ClauseResult struct { + Field string `json:"field"` + Op string `json:"op"` + EventValue string `json:"event_value,omitempty"` + Present bool `json:"present"` + Matched bool `json:"matched"` +} + +// ConditionResult is the outcome of one if-condition against current state. +type ConditionResult struct { + Kind string `json:"kind"` + Matched bool `json:"matched"` + Detail string `json:"detail,omitempty"` +} + +// Evaluation is the complete read-only decision for one (event, revision) pair. +type Evaluation struct { + Event string `json:"event"` + HookEvent string `json:"hook_event"` + FireMode string `json:"fire_mode"` + Matched bool `json:"matched"` + MatchClauses []ClauseResult `json:"match_clauses"` + ConditionsMet bool `json:"conditions_met"` + Conditions []ConditionResult `json:"conditions"` + WouldFire bool `json:"would_fire"` + Reason string `json:"reason"` + EvaluatedAgainst string `json:"evaluated_against"` + Note string `json:"note,omitempty"` +} + +// Evaluate is the shared read-only decision. It never mutates durable state. +func Evaluate(ctx context.Context, event EventView, rev EvalRevision, state StateReader) (Evaluation, error) { + ev := Evaluation{ + Event: event.Type, + HookEvent: rev.EventType, + FireMode: rev.FireMode, + EvaluatedAgainst: EvaluatedAgainstCurrentState, + } + + // A revision only matches events of its own type. + if event.Type != rev.EventType { + ev.Reason = ReasonEventTypeMismatch + return ev, nil + } + + match, err := ParseMatch(rev.Match) + if err != nil { + return Evaluation{}, fmt.Errorf("parse stored match: %w", err) + } + ev.Matched, ev.MatchClauses = evalMatch(event, match) + if !ev.Matched { + ev.Reason = ReasonNoMatch + return ev, nil + } + + ev.ConditionsMet = true + for _, c := range rev.Conditions { + cr, err := evalCondition(ctx, c, state) + if err != nil { + return Evaluation{}, err + } + ev.Conditions = append(ev.Conditions, cr) + if !cr.Matched { + ev.ConditionsMet = false + } + } + if !ev.ConditionsMet { + ev.Reason = ReasonConditionFalse + return ev, nil + } + + ev.Reason = ReasonMatched + ev.WouldFire = true + if rev.FireMode == FireRisingEdge { + // Honest read-only note (2A): the durable latch decides the actual edge. + ev.Note = "rising_edge fires only on a false→true edge; the durable latch is not consulted in read-only dry-run/explain" + } + return ev, nil +} + +// field resolves an event field path to a normalized string value. +func (e EventView) field(path string) (string, bool) { + switch path { + case "subject_id": + return e.SubjectID, e.SubjectID != "" + case "actor_type": + return e.ActorType, e.ActorType != "" + case "actor_id": + return e.ActorID, e.ActorID != "" + } + raw, ok := e.Payload[path] + if !ok { + return "", false + } + return normalizeScalar(raw) +} + +func evalMatch(event EventView, m Match) (bool, []ClauseResult) { + matched := true + results := make([]ClauseResult, 0, len(m)) + for field, clause := range m { + val, present := event.field(field) + ok := evalClause(clause, val, present) + if !ok { + matched = false + } + results = append(results, ClauseResult{ + Field: field, + Op: string(clause.Op), + EventValue: val, + Present: present, + Matched: ok, + }) + } + // Stable order for deterministic snapshots / responses. + sort.Slice(results, func(i, j int) bool { return results[i].Field < results[j].Field }) + return matched, results +} + +func evalClause(c MatchClause, val string, present bool) bool { + switch c.Op { + case MatchEq: + return present && val == c.Value + case MatchIn: + return present && contains(c.Set, val) + case MatchExists: + return present == c.Exists + } + return false +} + +func evalCondition(ctx context.Context, c ConditionSpec, state StateReader) (ConditionResult, error) { + if c.IssuesStatus != nil { + return evalIssuesStatus(ctx, *c.IssuesStatus, state) + } + if c.IssueField != nil { + return evalIssueField(ctx, *c.IssueField, state) + } + // A stored, validated condition always has exactly one variant. + return ConditionResult{Kind: "unknown", Matched: false, Detail: "empty condition"}, nil +} + +func evalIssuesStatus(ctx context.Context, c IssuesStatusCond, state StateReader) (ConditionResult, error) { + target, mode := c.All, "all" + if c.Any != "" { + target, mode = c.Any, "any" + } + allHit, anyHit := true, false + for _, id := range c.IDs { + status, exists, err := state.IssueField(ctx, id, IssueFieldStatus) + if err != nil { + return ConditionResult{}, err + } + if exists && status == target { + anyHit = true + } else { + allHit = false + } + } + met := allHit + if mode == "any" { + met = anyHit + } + return ConditionResult{ + Kind: "issues_status", + Matched: met, + Detail: fmt.Sprintf("%s of %d issues == %q", mode, len(c.IDs), target), + }, nil +} + +func evalIssueField(ctx context.Context, c IssueFieldCond, state StateReader) (ConditionResult, error) { + val, exists, err := state.IssueField(ctx, c.ID, c.Field) + if err != nil { + return ConditionResult{}, err + } + met := false + if exists { + if c.Eq != "" { + met = val == c.Eq + } else { + met = contains(c.In, val) + } + } + return ConditionResult{ + Kind: "issue_field", + Matched: met, + Detail: fmt.Sprintf("issue %s.%s", c.ID, c.Field), + }, nil +} + +func normalizeScalar(v any) (string, bool) { + switch t := v.(type) { + case string: + return t, true + case bool: + return fmt.Sprintf("%t", t), true + case float64: + return formatNumber(t), true + default: + // null, arrays and objects are not matchable scalars. + return "", false + } +} + +func contains(set []string, v string) bool { + for _, s := range set { + if s == v { + return true + } + } + return false +} diff --git a/server/internal/automation/eval_test.go b/server/internal/automation/eval_test.go new file mode 100644 index 00000000000..4cae22695b0 --- /dev/null +++ b/server/internal/automation/eval_test.go @@ -0,0 +1,147 @@ +package automation + +import ( + "context" + "encoding/json" + "testing" +) + +// fakeState is an in-memory StateReader: issueID -> field -> value. +type fakeState map[string]map[string]string + +func (f fakeState) IssueField(_ context.Context, id, field string) (string, bool, error) { + m, ok := f[id] + if !ok { + return "", false, nil + } + v, ok := m[field] + return v, ok, nil +} + +func statusChangedEvent(subjectID, from, to string) EventView { + return EventView{ + Type: "issue.status_changed", + SubjectID: subjectID, + ActorType: "member", + ActorID: uuidM, + Payload: map[string]any{"from": from, "to": to}, + } +} + +func TestEvaluateEventTypeMismatch(t *testing.T) { + ev, err := Evaluate(context.Background(), + EventView{Type: "comment.created"}, + EvalRevision{EventType: "issue.status_changed", FireMode: FirePerEvent}, + fakeState{}) + if err != nil { + t.Fatal(err) + } + if ev.Reason != ReasonEventTypeMismatch || ev.Matched || ev.WouldFire { + t.Fatalf("unexpected: %+v", ev) + } +} + +func TestEvaluateMatchClauses(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done","subject_id":{"in":["` + uuidA + `"]},"from":{"exists":true}}`), + FireMode: FirePerEvent, + } + ev, err := Evaluate(context.Background(), event, rev, fakeState{}) + if err != nil { + t.Fatal(err) + } + if !ev.Matched || !ev.WouldFire || ev.Reason != ReasonMatched { + t.Fatalf("expected match, got %+v", ev) + } + if len(ev.MatchClauses) != 3 { + t.Errorf("clauses = %d, want 3", len(ev.MatchClauses)) + } + + // A wrong `to` value fails the match. + rev.Match = json.RawMessage(`{"to":"blocked"}`) + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.Matched || ev.Reason != ReasonNoMatch { + t.Errorf("expected no_match, got %+v", ev) + } + + // exists:false on a present field fails. + rev.Match = json.RawMessage(`{"to":{"exists":false}}`) + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.Matched { + t.Errorf("exists:false on a present field should not match") + } +} + +func TestEvaluateConditionsAgainstCurrentState(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + } + // Both done → conditions met. + state := fakeState{uuidA: {"status": "done"}, uuidB: {"status": "done"}} + ev, err := Evaluate(context.Background(), event, rev, state) + if err != nil { + t.Fatal(err) + } + if !ev.ConditionsMet || ev.Reason != ReasonMatched || ev.EvaluatedAgainst != EvaluatedAgainstCurrentState { + t.Fatalf("expected conditions met against current state, got %+v", ev) + } + + // B not done → all-condition fails. + state[uuidB] = map[string]string{"status": "todo"} + ev, _ = Evaluate(context.Background(), event, rev, state) + if ev.ConditionsMet || ev.Reason != ReasonConditionFalse || ev.WouldFire { + t.Fatalf("expected condition_false, got %+v", ev) + } + + // A missing issue is treated as unsatisfied for `all`. + rev.Conditions = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidC}, All: "done"}}} + ev, _ = Evaluate(context.Background(), event, rev, fakeState{}) + if ev.ConditionsMet { + t.Errorf("missing issue must not satisfy an all-condition") + } + + // `any` semantics: at least one match suffices. + rev.Conditions = []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, Any: "done"}}} + ev, _ = Evaluate(context.Background(), event, rev, fakeState{uuidA: {"status": "done"}, uuidB: {"status": "todo"}}) + if !ev.ConditionsMet { + t.Errorf("any-condition should be met when one issue matches") + } +} + +func TestEvaluateIssueFieldCondition(t *testing.T) { + event := statusChangedEvent(uuidA, "todo", "in_progress") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssueField: &IssueFieldCond{ID: uuidA, Field: "assignee_id", Eq: uuidM}}}, + } + ev, _ := Evaluate(context.Background(), event, rev, fakeState{uuidA: {"assignee_id": uuidM}}) + if !ev.ConditionsMet { + t.Errorf("issue_field eq should match") + } + ev, _ = Evaluate(context.Background(), event, rev, fakeState{uuidA: {"assignee_id": uuidB}}) + if ev.ConditionsMet { + t.Errorf("issue_field eq should not match a different assignee") + } +} + +func TestEvaluateRisingEdgeNote(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FireRisingEdge, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA}, All: "done"}}}, + } + ev, _ := Evaluate(context.Background(), event, rev, fakeState{uuidA: {"status": "done"}}) + if ev.Reason != ReasonMatched || ev.Note == "" { + t.Fatalf("rising_edge matched should carry a read-only latch note, got %+v", ev) + } +} diff --git a/server/internal/handler/hook.go b/server/internal/handler/hook.go index 9ff499c5ac1..46481963788 100644 --- a/server/internal/handler/hook.go +++ b/server/internal/handler/hook.go @@ -383,6 +383,8 @@ func (h *Handler) writeHookError(w http.ResponseWriter, err error) { switch { case errors.Is(err, service.ErrHookNotFound): writeError(w, http.StatusNotFound, "hook not found") + case errors.Is(err, service.ErrHookEventNotFound): + writeError(w, http.StatusNotFound, "event not found") case errors.Is(err, service.ErrHookSystemManaged): writeError(w, http.StatusForbidden, err.Error()) case errors.Is(err, service.ErrHookForbidden): diff --git a/server/internal/handler/hook_eval.go b/server/internal/handler/hook_eval.go new file mode 100644 index 00000000000..4668e571d66 --- /dev/null +++ b/server/internal/handler/hook_eval.go @@ -0,0 +1,161 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/multica-ai/multica/server/internal/automation" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// hookEventListLimit is defensive; a correlation chain is bounded by the depth / +// width guardrails, but cap the debug read anyway. +const hookEventListLimit = 1000 + +// DryRunHookRequest dry-runs a candidate spec against a historical event. +type DryRunHookRequest struct { + Hook automation.HookSpec `json:"hook"` + EventID string `json:"event_id"` +} + +// ExplainHookRequest explains a stored hook revision's decision for an event. +type ExplainHookRequest struct { + HookID string `json:"hook_id"` + EventID string `json:"event_id"` + Revision int32 `json:"revision,omitempty"` +} + +// DomainEventResponse is the read-only API view of a domain event. +type DomainEventResponse struct { + ID string `json:"id"` + Seq int64 `json:"seq"` + Type string `json:"type"` + SubjectType string `json:"subject_type"` + SubjectID string `json:"subject_id"` + ActorType string `json:"actor_type"` + ActorID string `json:"actor_id,omitempty"` + Payload json.RawMessage `json:"payload"` + CorrelationID string `json:"correlation_id"` + CausationExecutionID string `json:"causation_execution_id,omitempty"` + HopCount int32 `json:"hop_count"` + CreatedAt string `json:"created_at"` +} + +// DryRunHook evaluates a candidate hook spec against a historical event without +// executing anything or mutating any durable state (§10, decision 2A). +func (h *Handler) DryRunHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + var req DryRunHookRequest + if !decodeJSONBodyStrict(w, r, &req) { + return + } + eventUUID, ok := parseUUIDOrBadRequest(w, req.EventID, "event_id") + if !ok { + return + } + result, err := h.HookService.DryRun(r.Context(), workspaceUUID, req.Hook, eventUUID) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// ExplainHook explains why a stored hook (its active or a specified revision) +// would or would not fire for a historical event. Read-only. +func (h *Handler) ExplainHook(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + var req ExplainHookRequest + if !decodeJSONBodyStrict(w, r, &req) { + return + } + hookUUID, ok := parseUUIDOrBadRequest(w, req.HookID, "hook_id") + if !ok { + return + } + eventUUID, ok := parseUUIDOrBadRequest(w, req.EventID, "event_id") + if !ok { + return + } + result, err := h.HookService.Explain(r.Context(), workspaceUUID, hookUUID, eventUUID, req.Revision) + if err != nil { + h.writeHookError(w, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +// ListEventsByCorrelation returns the domain events in a correlation chain, for +// execution-chain debugging (GET /api/events?correlation_id=). Read-only. +func (h *Handler) ListEventsByCorrelation(w http.ResponseWriter, r *http.Request) { + if !h.hookEnabled(w, r) { + return + } + workspaceUUID, ok := parseUUIDOrBadRequest(w, h.resolveWorkspaceID(r), "workspace_id") + if !ok { + return + } + correlationUUID, ok := parseUUIDOrBadRequest(w, r.URL.Query().Get("correlation_id"), "correlation_id") + if !ok { + return + } + events, err := h.HookService.EventsByCorrelation(r.Context(), workspaceUUID, correlationUUID) + if err != nil { + h.writeHookError(w, err) + return + } + resp := make([]DomainEventResponse, 0, len(events)) + for i, e := range events { + if i >= hookEventListLimit { + break + } + resp = append(resp, domainEventToResponse(e)) + } + writeJSON(w, http.StatusOK, resp) +} + +func domainEventToResponse(e db.DomainEvent) DomainEventResponse { + return DomainEventResponse{ + ID: uuidToString(e.ID), + Seq: e.Seq, + Type: e.Type, + SubjectType: e.SubjectType, + SubjectID: uuidToString(e.SubjectID), + ActorType: e.ActorType, + ActorID: uuidToString(e.ActorID), + Payload: rawJSON(e.Payload), + CorrelationID: uuidToString(e.CorrelationID), + CausationExecutionID: uuidToString(e.CausationExecutionID), + HopCount: e.HopCount, + CreatedAt: timestampToString(e.CreatedAt), + } +} + +// decodeJSONBodyStrict decodes exactly one JSON document into dst, rejecting +// unknown fields and any trailing second document. +func decodeJSONBodyStrict(w http.ResponseWriter, r *http.Request, dst any) bool { + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error()) + return false + } + if err := dec.Decode(new(json.RawMessage)); err != io.EOF { + writeError(w, http.StatusBadRequest, "invalid request body: expected exactly one JSON document") + return false + } + return true +} diff --git a/server/internal/handler/hook_eval_test.go b/server/internal/handler/hook_eval_test.go new file mode 100644 index 00000000000..97dfd758da9 --- /dev/null +++ b/server/internal/handler/hook_eval_test.go @@ -0,0 +1,221 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/multica-ai/multica/server/internal/automation" +) + +// seedStatusChangedEvent inserts an issue.status_changed domain_event whose +// subject is issueID, with a from/to payload and the given correlation id. +func seedStatusChangedEvent(t *testing.T, issueID, from, to, correlationID string) string { + t.Helper() + var id string + payload := fmt.Sprintf(`{"from":%q,"to":%q}`, from, to) + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id) + VALUES ($1, 'issue.status_changed', 1, 'issue', $2, 'member', $3, $4::jsonb, $5) + RETURNING id`, + testWorkspaceID, issueID, testUserID, payload, correlationID).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + return id +} + +func decodeEvaluation(t *testing.T, w *httptest.ResponseRecorder) automation.Evaluation { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var ev automation.Evaluation + if err := json.NewDecoder(w.Body).Decode(&ev); err != nil { + t.Fatalf("decode evaluation: %v", err) + } + return ev +} + +func TestHookDryRun(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + eventID := seedStatusChangedEvent(t, issueID, "in_progress", "done", issueID) + + // A spec matching to=done fires; evaluated against current state. + w := httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": sampleHookSpec("dr", "hi", issueID), "event_id": eventID})) + ev := decodeEvaluation(t, w) + if !ev.Matched || !ev.WouldFire || ev.Reason != automation.ReasonMatched { + t.Fatalf("expected match, got %+v", ev) + } + if ev.EvaluatedAgainst != automation.EvaluatedAgainstCurrentState { + t.Errorf("evaluated_against = %q, want current_state", ev.EvaluatedAgainst) + } + + // A spec whose match requires to=blocked does not fire. + spec := sampleHookSpec("dr2", "hi", issueID) + spec["when"].(map[string]any)["match"] = map[string]any{"to": "blocked"} + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + ev = decodeEvaluation(t, w) + if ev.Matched || ev.Reason != automation.ReasonNoMatch { + t.Errorf("expected no_match, got %+v", ev) + } + + // A missing event is 404. + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": sampleHookSpec("dr3", "hi", issueID), "event_id": "99999999-9999-9999-9999-999999999999"})) + if w.Code != http.StatusNotFound { + t.Errorf("dry-run missing event: status %d, want 404", w.Code) + } + + // An invalid spec is 400 (shape validation), never reaching evaluation. + bad := sampleHookSpec("bad", "hi", issueID) + bad["do"] = []any{map[string]any{"type": "set_issue_status_many"}} + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": bad, "event_id": eventID})) + if w.Code != http.StatusBadRequest { + t.Errorf("dry-run invalid spec: status %d, want 400", w.Code) + } +} + +// dry-run reads conditions against CURRENT workspace state, not the event moment. +func TestHookDryRunConditionsUseCurrentState(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + ctx := context.Background() + issueID := seedHookIssue(t) // seeded as status 'todo' + eventID := seedStatusChangedEvent(t, issueID, "todo", "in_progress", issueID) + + spec := sampleHookSpec("cond", "hi", issueID) + spec["when"].(map[string]any)["match"] = map[string]any{} + spec["if"] = []any{map[string]any{"issues_status": map[string]any{"ids": []any{issueID}, "all": "done"}}} + + // Issue is currently 'todo' → condition (all done) is false. + w := httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + if ev := decodeEvaluation(t, w); ev.ConditionsMet || ev.Reason != automation.ReasonConditionFalse { + t.Fatalf("expected condition_false with issue todo, got %+v", ev) + } + + // Move the issue to done → the same dry-run now reports conditions met. + if _, err := testPool.Exec(ctx, `UPDATE issue SET status = 'done' WHERE id = $1`, issueID); err != nil { + t.Fatalf("update issue: %v", err) + } + w = httptest.NewRecorder() + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", + map[string]any{"hook": spec, "event_id": eventID})) + if ev := decodeEvaluation(t, w); !ev.ConditionsMet || ev.Reason != automation.ReasonMatched { + t.Fatalf("expected conditions met after issue moved to done, got %+v", ev) + } +} + +func TestHookExplain(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + hook := createHookAs(t, testUserID, sampleHookSpec("explain hook", "hi", issueID)) // matches to=done + + matchEvent := seedStatusChangedEvent(t, issueID, "in_progress", "done", issueID) + noMatchEvent := seedStatusChangedEvent(t, issueID, "in_progress", "todo", issueID) + + w := httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": hook.ID, "event_id": matchEvent})) + if ev := decodeEvaluation(t, w); ev.Reason != automation.ReasonMatched { + t.Errorf("explain matching event: reason %q, want matched (%+v)", ev.Reason, ev) + } + + w = httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": hook.ID, "event_id": noMatchEvent})) + if ev := decodeEvaluation(t, w); ev.Reason != automation.ReasonNoMatch { + t.Errorf("explain non-matching event: reason %q, want no_match", ev.Reason) + } + + // Unknown hook is 404. + w = httptest.NewRecorder() + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", + map[string]any{"hook_id": "88888888-8888-8888-8888-888888888888", "event_id": matchEvent})) + if w.Code != http.StatusNotFound { + t.Errorf("explain unknown hook: status %d, want 404", w.Code) + } +} + +func TestHookEventsByCorrelation(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + seedStatusChangedEvent(t, issueID, "todo", "in_progress", correlation) + seedStatusChangedEvent(t, issueID, "in_progress", "done", correlation) + // A different correlation must not leak in. + seedStatusChangedEvent(t, issueID, "todo", "done", "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 2 { + t.Fatalf("events = %d, want 2 (the correlation chain only)", len(events)) + } + for _, e := range events { + if e.CorrelationID != correlation { + t.Errorf("leaked event from correlation %s", e.CorrelationID) + } + } +} + +// The whole read-only surface is invisible unless the feature flag is on. +func TestHookEvalRequiresFeatureFlag(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + prev := testHandler.FeatureFlags + testHandler.FeatureFlags = nil + t.Cleanup(func() { testHandler.FeatureFlags = prev }) + + for _, tc := range []struct { + name string + call func(w *httptest.ResponseRecorder) + }{ + {"dry-run", func(w *httptest.ResponseRecorder) { + testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", map[string]any{})) + }}, + {"explain", func(w *httptest.ResponseRecorder) { + testHandler.ExplainHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/explain", map[string]any{})) + }}, + {"events", func(w *httptest.ResponseRecorder) { + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events", nil)) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + w := httptest.NewRecorder() + tc.call(w) + if w.Code != http.StatusNotFound { + t.Errorf("%s with flag off: status %d, want 404", tc.name, w.Code) + } + }) + } +} diff --git a/server/internal/service/hook_eval.go b/server/internal/service/hook_eval.go new file mode 100644 index 00000000000..6489ea85e17 --- /dev/null +++ b/server/internal/service/hook_eval.go @@ -0,0 +1,162 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// Read-only Event Hooks debug surface (MUL-4332 PR3, decision 2A): dry-run, +// explain and the correlation query. These reuse the single automation.Evaluate +// evaluator so an explanation can never drift from real execution, and they +// perform NO action and mutate NO durable state. + +// ErrHookEventNotFound is returned when a referenced domain event does not exist +// in the workspace. +var ErrHookEventNotFound = errors.New("event not found") + +// issueStateReader is the workspace-scoped StateReader the evaluator reads +// current issue state through. +type issueStateReader struct { + q *db.Queries + workspaceID pgtype.UUID +} + +func (r *issueStateReader) IssueField(ctx context.Context, issueID, field string) (string, bool, error) { + uid, err := util.ParseUUID(issueID) + if err != nil { + return "", false, nil // a malformed id can never resolve to a workspace issue + } + issue, err := r.q.GetIssueInWorkspace(ctx, db.GetIssueInWorkspaceParams{ID: uid, WorkspaceID: r.workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + return "", false, err + } + switch field { + case automation.IssueFieldStatus: + return issue.Status, true, nil + case automation.IssueFieldAssigneeID: + return util.UUIDToString(issue.AssigneeID), issue.AssigneeID.Valid, nil + case automation.IssueFieldParentIssueID: + return util.UUIDToString(issue.ParentIssueID), issue.ParentIssueID.Valid, nil + } + return "", false, nil +} + +// DryRun evaluates a candidate hook spec against a historical event, read-only. +// The spec's shape is validated (so garbage is rejected early), the event's +// `when` is matched against its historical payload, and `if` conditions read +// current workspace state. +func (s *HookService) DryRun(ctx context.Context, workspaceID pgtype.UUID, spec automation.HookSpec, eventID pgtype.UUID) (automation.Evaluation, error) { + if err := automation.Validate(spec); err != nil { + return automation.Evaluation{}, err + } + view, err := s.loadEventView(ctx, workspaceID, eventID) + if err != nil { + return automation.Evaluation{}, err + } + rev := automation.EvalRevision{ + EventType: spec.When.Event, + Match: spec.When.Match, + Conditions: spec.If, + FireMode: spec.Fire.Mode, + } + return automation.Evaluate(ctx, view, rev, &issueStateReader{q: s.Queries, workspaceID: workspaceID}) +} + +// Explain evaluates a stored hook's revision against a historical event, +// read-only. revisionNumber == 0 explains the active revision. +func (s *HookService) Explain(ctx context.Context, workspaceID, hookID, eventID pgtype.UUID, revisionNumber int32) (automation.Evaluation, error) { + hook, err := s.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: hookID, WorkspaceID: workspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.Evaluation{}, ErrHookNotFound + } + return automation.Evaluation{}, err + } + var rawRev db.HookRevision + if revisionNumber > 0 { + rawRev, err = s.Queries.GetHookRevisionByNumber(ctx, db.GetHookRevisionByNumberParams{HookID: hookID, Revision: revisionNumber}) + } else { + rawRev, err = s.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + } + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.Evaluation{}, ErrHookNotFound + } + return automation.Evaluation{}, err + } + + view, err := s.loadEventView(ctx, workspaceID, eventID) + if err != nil { + return automation.Evaluation{}, err + } + rev, err := revisionToEval(rawRev) + if err != nil { + return automation.Evaluation{}, err + } + return automation.Evaluate(ctx, view, rev, &issueStateReader{q: s.Queries, workspaceID: workspaceID}) +} + +// EventsByCorrelation returns the domain events in a correlation chain (ordered +// by seq), for execution-chain debugging. +func (s *HookService) EventsByCorrelation(ctx context.Context, workspaceID, correlationID pgtype.UUID) ([]db.DomainEvent, error) { + return s.Queries.ListDomainEventsByCorrelation(ctx, db.ListDomainEventsByCorrelationParams{ + WorkspaceID: workspaceID, CorrelationID: correlationID, + }) +} + +func (s *HookService) loadEventView(ctx context.Context, workspaceID, eventID pgtype.UUID) (automation.EventView, error) { + event, err := s.Queries.GetDomainEvent(ctx, eventID) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return automation.EventView{}, ErrHookEventNotFound + } + return automation.EventView{}, err + } + // GetDomainEvent is not workspace-scoped; enforce tenant isolation here. + if !principalMatches(event.WorkspaceID, workspaceID) { + return automation.EventView{}, ErrHookEventNotFound + } + return eventToView(event) +} + +func eventToView(e db.DomainEvent) (automation.EventView, error) { + var payload map[string]any + if len(e.Payload) > 0 { + if err := json.Unmarshal(e.Payload, &payload); err != nil { + return automation.EventView{}, err + } + } + return automation.EventView{ + Type: e.Type, + SubjectID: util.UUIDToString(e.SubjectID), + ActorType: e.ActorType, + ActorID: util.UUIDToString(e.ActorID), + Payload: payload, + }, nil +} + +func revisionToEval(rev db.HookRevision) (automation.EvalRevision, error) { + var conds []automation.ConditionSpec + if len(rev.Conditions) > 0 { + if err := json.Unmarshal(rev.Conditions, &conds); err != nil { + return automation.EvalRevision{}, err + } + } + return automation.EvalRevision{ + EventType: rev.EventType, + Match: rev.Match, + Conditions: conds, + FireMode: rev.FireMode, + }, nil +} diff --git a/server/pkg/db/generated/hook.sql.go b/server/pkg/db/generated/hook.sql.go index 7f51bf31fad..f7c604c30d4 100644 --- a/server/pkg/db/generated/hook.sql.go +++ b/server/pkg/db/generated/hook.sql.go @@ -280,6 +280,36 @@ func (q *Queries) GetHookRevision(ctx context.Context, id pgtype.UUID) (HookRevi return i, err } +const getHookRevisionByNumber = `-- name: GetHookRevisionByNumber :one +SELECT id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id, created_at FROM hook_revision +WHERE hook_id = $1 AND revision = $2 +` + +type GetHookRevisionByNumberParams struct { + HookID pgtype.UUID `json:"hook_id"` + Revision int32 `json:"revision"` +} + +// A specific revision of a hook, for `explain --revision N` (read-only debug). +func (q *Queries) GetHookRevisionByNumber(ctx context.Context, arg GetHookRevisionByNumberParams) (HookRevision, error) { + row := q.db.QueryRow(ctx, getHookRevisionByNumber, arg.HookID, arg.Revision) + var i HookRevision + err := row.Scan( + &i.ID, + &i.HookID, + &i.Revision, + &i.EventType, + &i.Match, + &i.Conditions, + &i.FireMode, + &i.Actions, + &i.CreatedByType, + &i.CreatedByID, + &i.CreatedAt, + ) + return i, err +} + const getMaxHookRevision = `-- name: GetMaxHookRevision :one SELECT COALESCE(MAX(revision), 0)::int AS max_revision FROM hook_revision diff --git a/server/pkg/db/queries/hook.sql b/server/pkg/db/queries/hook.sql index a9d0f4e26d1..9ddc88d0bc2 100644 --- a/server/pkg/db/queries/hook.sql +++ b/server/pkg/db/queries/hook.sql @@ -83,6 +83,11 @@ RETURNING *; SELECT * FROM hook_revision WHERE id = $1; +-- name: GetHookRevisionByNumber :one +-- A specific revision of a hook, for `explain --revision N` (read-only debug). +SELECT * FROM hook_revision +WHERE hook_id = $1 AND revision = $2; + -- name: GetMaxHookRevision :one -- Highest revision number for a hook, 0 when none exist yet. Used to compute the -- next revision on PATCH. From 3e12c570edccf3113705257b4a2936e85c352736 Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Fri, 17 Jul 2026 17:22:05 +0800 Subject: [PATCH 13/15] fix(automation): tri-state fire + structured snapshot + safe correlation (MUL-4332 PR3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Elon's round-1 review on the PR3 read-only slice. 1. would_fire no longer over-claims. The evaluator set would_fire=true after match+condition even though the rising-edge latch and the depth/rate/permission/ fuse guards are not evaluated in read-only mode. Replaced with Eligible (match held AND conditions currently satisfied) + DecisionComplete (false in read-only mode). No field claims execution until the matcher (next slice) evaluates the latch + guards and sets DecisionComplete. 2. Evaluator output is now a shareable structured snapshot. ClauseResult carries op + observed + present + expected; ConditionResult carries mode/field/op/ expected plus per-issue observed status/presence/result. Added MatchSnapshot / ConditionSnapshot helpers that serialize exactly what the matcher will store, so dry-run, explain and (next slice) the execution snapshot consume one result with no re-read and no parallel logic — the drift 2A forbids. 3. Correlation API is fail-closed and locatable. Payload is projected to the event type's declared schema fields (automation.ProjectPayload drops free-text / undeclared keys, e.g. an issue title; unknown types → empty); the response adds schema_version and causation_action_index; and the chain LIMIT is pushed into the (workspace_id, correlation_id, seq) query instead of truncating a fully loaded chain. Tests: evaluator structured-snapshot assertion + tri-state fields; ProjectPayload unit (redaction + unknown-type fail-closed); correlation payload redaction, foreign-workspace same-correlation filtered, and SQL-limit pushdown. No migration change; one query gains a LIMIT param. The matcher wiring to the same evaluator and the (event, revision, state) snapshot- consistency regression are the next PR3 slice. Co-authored-by: multica-agent --- server/internal/automation/eval.go | 120 +++++++++++++++----- server/internal/automation/eval_test.go | 78 ++++++++++++- server/internal/automation/schema.go | 33 ++++++ server/internal/handler/hook_eval.go | 31 +++-- server/internal/handler/hook_eval_test.go | 95 +++++++++++++++- server/internal/service/hook_eval.go | 9 +- server/pkg/db/generated/domain_event.sql.go | 7 +- server/pkg/db/queries/domain_event.sql | 6 +- 8 files changed, 334 insertions(+), 45 deletions(-) diff --git a/server/internal/automation/eval.go b/server/internal/automation/eval.go index b013f662f39..cf794100069 100644 --- a/server/internal/automation/eval.go +++ b/server/internal/automation/eval.go @@ -57,23 +57,50 @@ type EvalRevision struct { FireMode string } -// ClauseResult is the per-field outcome of one when-match clause. +// ClauseResult is the per-field outcome of one when-match clause. It records the +// operator, the observed event value and presence, and the expected operand, so a +// single structured result feeds dry-run, explain and (next slice) the matcher's +// stored match_snapshot — no re-derivation, no parallel logic (review point 2). type ClauseResult struct { - Field string `json:"field"` - Op string `json:"op"` - EventValue string `json:"event_value,omitempty"` - Present bool `json:"present"` - Matched bool `json:"matched"` + Field string `json:"field"` + Op string `json:"op"` + Observed string `json:"observed,omitempty"` + Present bool `json:"present"` + Expected []string `json:"expected,omitempty"` + Matched bool `json:"matched"` } -// ConditionResult is the outcome of one if-condition against current state. +// IssueObserved records the observed current value of one issue field consulted +// by a condition, and whether that issue satisfied the predicate. +type IssueObserved struct { + ID string `json:"id"` + Field string `json:"field"` + Observed string `json:"observed,omitempty"` + Present bool `json:"present"` + Matched bool `json:"matched"` +} + +// ConditionResult is the structured outcome of one if-condition against current +// state: the operator/expected operand plus every observed input, so the same +// object is the condition_snapshot the matcher will store (review point 2). type ConditionResult struct { - Kind string `json:"kind"` - Matched bool `json:"matched"` - Detail string `json:"detail,omitempty"` + Kind string `json:"kind"` // issues_status | issue_field + Matched bool `json:"matched"` + Mode string `json:"mode,omitempty"` // all | any (issues_status) + Field string `json:"field,omitempty"` // issue field (issue_field) + Op string `json:"op,omitempty"` // eq | in (issue_field) + Expected []string `json:"expected,omitempty"` // target status / eq value / in-set + Issues []IssueObserved `json:"issues,omitempty"` // per-issue observed state } // Evaluation is the complete read-only decision for one (event, revision) pair. +// +// Eligible means the when-match held and every if-condition is currently +// satisfied. It is NOT a fire decision: a rising-edge latch and the depth / rate +// / permission / fuse guards are not evaluated in read-only mode, so +// DecisionComplete is false and no field ever claims the rule "will execute" +// (review point 1). The matcher (next slice) will set DecisionComplete and the +// final fire verdict after evaluating the latch and guards. type Evaluation struct { Event string `json:"event"` HookEvent string `json:"hook_event"` @@ -82,12 +109,19 @@ type Evaluation struct { MatchClauses []ClauseResult `json:"match_clauses"` ConditionsMet bool `json:"conditions_met"` Conditions []ConditionResult `json:"conditions"` - WouldFire bool `json:"would_fire"` + Eligible bool `json:"eligible"` + DecisionComplete bool `json:"decision_complete"` Reason string `json:"reason"` EvaluatedAgainst string `json:"evaluated_against"` Note string `json:"note,omitempty"` } +// MatchSnapshot / ConditionSnapshot serialize the structured inputs+conclusions +// exactly as the matcher will persist them, so dry-run/explain and the stored +// execution snapshot are byte-identical for the same (event, revision, state). +func (e Evaluation) MatchSnapshot() (json.RawMessage, error) { return json.Marshal(e.MatchClauses) } +func (e Evaluation) ConditionSnapshot() (json.RawMessage, error) { return json.Marshal(e.Conditions) } + // Evaluate is the shared read-only decision. It never mutates durable state. func Evaluate(ctx context.Context, event EventView, rev EvalRevision, state StateReader) (Evaluation, error) { ev := Evaluation{ @@ -129,11 +163,16 @@ func Evaluate(ctx context.Context, event EventView, rev EvalRevision, state Stat return ev, nil } + // The event matches and conditions currently hold. This makes the rule + // ELIGIBLE, but not a completed fire decision: read-only evaluation does not + // consult the rising-edge latch or the depth/rate/permission/fuse guards, so + // DecisionComplete stays false and no field claims execution (review point 1). + ev.Eligible = true ev.Reason = ReasonMatched - ev.WouldFire = true if rev.FireMode == FireRisingEdge { - // Honest read-only note (2A): the durable latch decides the actual edge. - ev.Note = "rising_edge fires only on a false→true edge; the durable latch is not consulted in read-only dry-run/explain" + ev.Note = "eligible now, but rising_edge fires only on a false→true edge; the durable latch and guards are not evaluated in read-only mode" + } else { + ev.Note = "eligible now; the depth/rate/permission/fuse guards are not evaluated in read-only mode" } return ev, nil } @@ -165,11 +204,12 @@ func evalMatch(event EventView, m Match) (bool, []ClauseResult) { matched = false } results = append(results, ClauseResult{ - Field: field, - Op: string(clause.Op), - EventValue: val, - Present: present, - Matched: ok, + Field: field, + Op: string(clause.Op), + Observed: val, + Present: present, + Expected: clauseExpected(clause), + Matched: ok, }) } // Stable order for deterministic snapshots / responses. @@ -189,6 +229,18 @@ func evalClause(c MatchClause, val string, present bool) bool { return false } +func clauseExpected(c MatchClause) []string { + switch c.Op { + case MatchEq: + return []string{c.Value} + case MatchIn: + return c.Set + case MatchExists: + return []string{fmt.Sprintf("%t", c.Exists)} + } + return nil +} + func evalCondition(ctx context.Context, c ConditionSpec, state StateReader) (ConditionResult, error) { if c.IssuesStatus != nil { return evalIssuesStatus(ctx, *c.IssuesStatus, state) @@ -197,7 +249,7 @@ func evalCondition(ctx context.Context, c ConditionSpec, state StateReader) (Con return evalIssueField(ctx, *c.IssueField, state) } // A stored, validated condition always has exactly one variant. - return ConditionResult{Kind: "unknown", Matched: false, Detail: "empty condition"}, nil + return ConditionResult{Kind: "unknown", Matched: false}, nil } func evalIssuesStatus(ctx context.Context, c IssuesStatusCond, state StateReader) (ConditionResult, error) { @@ -206,25 +258,32 @@ func evalIssuesStatus(ctx context.Context, c IssuesStatusCond, state StateReader target, mode = c.Any, "any" } allHit, anyHit := true, false + observed := make([]IssueObserved, 0, len(c.IDs)) for _, id := range c.IDs { status, exists, err := state.IssueField(ctx, id, IssueFieldStatus) if err != nil { return ConditionResult{}, err } - if exists && status == target { + hit := exists && status == target + if hit { anyHit = true } else { allHit = false } + observed = append(observed, IssueObserved{ID: id, Field: IssueFieldStatus, Observed: status, Present: exists, Matched: hit}) } met := allHit if mode == "any" { met = anyHit } return ConditionResult{ - Kind: "issues_status", - Matched: met, - Detail: fmt.Sprintf("%s of %d issues == %q", mode, len(c.IDs), target), + Kind: "issues_status", + Matched: met, + Mode: mode, + Field: IssueFieldStatus, + Op: string(MatchEq), + Expected: []string{target}, + Issues: observed, }, nil } @@ -233,7 +292,11 @@ func evalIssueField(ctx context.Context, c IssueFieldCond, state StateReader) (C if err != nil { return ConditionResult{}, err } + op, expected := string(MatchEq), []string{c.Eq} met := false + if c.Eq == "" { + op, expected = string(MatchIn), c.In + } if exists { if c.Eq != "" { met = val == c.Eq @@ -242,9 +305,12 @@ func evalIssueField(ctx context.Context, c IssueFieldCond, state StateReader) (C } } return ConditionResult{ - Kind: "issue_field", - Matched: met, - Detail: fmt.Sprintf("issue %s.%s", c.ID, c.Field), + Kind: "issue_field", + Matched: met, + Field: c.Field, + Op: op, + Expected: expected, + Issues: []IssueObserved{{ID: c.ID, Field: c.Field, Observed: val, Present: exists, Matched: met}}, }, nil } diff --git a/server/internal/automation/eval_test.go b/server/internal/automation/eval_test.go index 4cae22695b0..61145ab31f0 100644 --- a/server/internal/automation/eval_test.go +++ b/server/internal/automation/eval_test.go @@ -36,7 +36,7 @@ func TestEvaluateEventTypeMismatch(t *testing.T) { if err != nil { t.Fatal(err) } - if ev.Reason != ReasonEventTypeMismatch || ev.Matched || ev.WouldFire { + if ev.Reason != ReasonEventTypeMismatch || ev.Matched || ev.Eligible { t.Fatalf("unexpected: %+v", ev) } } @@ -52,7 +52,7 @@ func TestEvaluateMatchClauses(t *testing.T) { if err != nil { t.Fatal(err) } - if !ev.Matched || !ev.WouldFire || ev.Reason != ReasonMatched { + if !ev.Matched || !ev.Eligible || ev.DecisionComplete || ev.Reason != ReasonMatched { t.Fatalf("expected match, got %+v", ev) } if len(ev.MatchClauses) != 3 { @@ -95,7 +95,7 @@ func TestEvaluateConditionsAgainstCurrentState(t *testing.T) { // B not done → all-condition fails. state[uuidB] = map[string]string{"status": "todo"} ev, _ = Evaluate(context.Background(), event, rev, state) - if ev.ConditionsMet || ev.Reason != ReasonConditionFalse || ev.WouldFire { + if ev.ConditionsMet || ev.Reason != ReasonConditionFalse || ev.Eligible { t.Fatalf("expected condition_false, got %+v", ev) } @@ -132,6 +132,78 @@ func TestEvaluateIssueFieldCondition(t *testing.T) { } } +// The evaluator output must carry observed + expected + op + present so a matcher +// can store one condition/match snapshot without re-reading state (review point 2). +func TestEvaluateStructuredSnapshot(t *testing.T) { + event := statusChangedEvent(uuidA, "in_progress", "done") + rev := EvalRevision{ + EventType: "issue.status_changed", + Match: json.RawMessage(`{"to":"done"}`), + FireMode: FirePerEvent, + Conditions: []ConditionSpec{{IssuesStatus: &IssuesStatusCond{IDs: []string{uuidA, uuidB}, All: "done"}}}, + } + state := fakeState{uuidA: {"status": "done"}, uuidB: {"status": "todo"}} + ev, err := Evaluate(context.Background(), event, rev, state) + if err != nil { + t.Fatal(err) + } + + // The match clause records op + observed + expected. + if len(ev.MatchClauses) != 1 { + t.Fatalf("match clauses = %d, want 1", len(ev.MatchClauses)) + } + mc := ev.MatchClauses[0] + if mc.Field != "to" || mc.Op != string(MatchEq) || mc.Observed != "done" || !mc.Present || len(mc.Expected) != 1 || mc.Expected[0] != "done" || !mc.Matched { + t.Errorf("match clause not fully structured: %+v", mc) + } + + // The condition records mode + expected + per-issue observed status. + if len(ev.Conditions) != 1 { + t.Fatalf("conditions = %d, want 1", len(ev.Conditions)) + } + c := ev.Conditions[0] + if c.Kind != "issues_status" || c.Mode != "all" || len(c.Expected) != 1 || c.Expected[0] != "done" || len(c.Issues) != 2 { + t.Fatalf("condition not fully structured: %+v", c) + } + byID := map[string]IssueObserved{} + for _, io := range c.Issues { + byID[io.ID] = io + } + if a := byID[uuidA]; a.Observed != "done" || !a.Present || !a.Matched { + t.Errorf("issue A observed wrong: %+v", a) + } + if b := byID[uuidB]; b.Observed != "todo" || !b.Present || b.Matched { + t.Errorf("issue B observed wrong: %+v", b) + } + + // Snapshots serialize the same structured inputs the matcher will store. + ms, err := ev.MatchSnapshot() + if err != nil || len(ms) == 0 { + t.Fatalf("match snapshot: %v", err) + } + cs, err := ev.ConditionSnapshot() + if err != nil || len(cs) == 0 { + t.Fatalf("condition snapshot: %v", err) + } +} + +// ProjectPayload is the fail-closed redaction for the correlation debug view. +func TestProjectPayload(t *testing.T) { + p := ProjectPayload("issue.created", map[string]any{"status": "todo", "title": "secret", "priority": "high", "bogus": 1}) + if _, ok := p["title"]; ok { + t.Error("free-text title must be redacted") + } + if _, ok := p["bogus"]; ok { + t.Error("undeclared field must be dropped") + } + if p["status"] != "todo" || p["priority"] != "high" { + t.Errorf("declared fields dropped: %v", p) + } + if len(ProjectPayload("issue.exploded", map[string]any{"x": 1})) != 0 { + t.Error("an unknown event type must project to an empty payload (fail-closed)") + } +} + func TestEvaluateRisingEdgeNote(t *testing.T) { event := statusChangedEvent(uuidA, "in_progress", "done") rev := EvalRevision{ diff --git a/server/internal/automation/schema.go b/server/internal/automation/schema.go index 61f300bf783..364e3a261df 100644 --- a/server/internal/automation/schema.go +++ b/server/internal/automation/schema.go @@ -91,6 +91,39 @@ func SchemaFor(eventType string) (EventSchema, bool) { return s, ok } +// PayloadFields returns the declared payload field names for an event type — its +// schema's match fields minus the common envelope fields. An unknown event type +// returns an empty set. Free-text / undeclared payload keys (e.g. an issue title) +// are intentionally absent, so a projection over this set is fail-closed. +func PayloadFields(eventType string) map[string]bool { + schema, ok := eventSchemas[eventType] + if !ok { + return map[string]bool{} + } + out := make(map[string]bool, len(schema.MatchFields)) + for f := range schema.MatchFields { + if _, isEnvelope := envelopeMatchFields[f]; !isEnvelope { + out[f] = true + } + } + return out +} + +// ProjectPayload returns a copy of payload keeping ONLY the event type's declared +// payload fields (fail-closed redaction, §10): undeclared, sensitive or free-text +// keys are dropped, and an unknown event type yields an empty object. Used to +// project a domain event's payload for the read-only correlation debug surface. +func ProjectPayload(eventType string, payload map[string]any) map[string]any { + allowed := PayloadFields(eventType) + out := make(map[string]any, len(allowed)) + for k, v := range payload { + if allowed[k] { + out[k] = v + } + } + return out +} + // Action types. User actions are creatable through the public API; system-only // actions are reserved for managed system hooks (PR5) and are rejected on a // user-authored spec. diff --git a/server/internal/handler/hook_eval.go b/server/internal/handler/hook_eval.go index 4668e571d66..1d95f67c174 100644 --- a/server/internal/handler/hook_eval.go +++ b/server/internal/handler/hook_eval.go @@ -26,11 +26,14 @@ type ExplainHookRequest struct { Revision int32 `json:"revision,omitempty"` } -// DomainEventResponse is the read-only API view of a domain event. +// DomainEventResponse is the read-only API view of a domain event. Payload is +// projected to the event's declared schema fields (fail-closed redaction); the +// causation_* fields locate the direct action that produced the event. type DomainEventResponse struct { ID string `json:"id"` Seq int64 `json:"seq"` Type string `json:"type"` + SchemaVersion int32 `json:"schema_version"` SubjectType string `json:"subject_type"` SubjectID string `json:"subject_id"` ActorType string `json:"actor_type"` @@ -38,6 +41,7 @@ type DomainEventResponse struct { Payload json.RawMessage `json:"payload"` CorrelationID string `json:"correlation_id"` CausationExecutionID string `json:"causation_execution_id,omitempty"` + CausationActionIndex *int32 `json:"causation_action_index,omitempty"` HopCount int32 `json:"hop_count"` CreatedAt string `json:"created_at"` } @@ -112,36 +116,47 @@ func (h *Handler) ListEventsByCorrelation(w http.ResponseWriter, r *http.Request if !ok { return } - events, err := h.HookService.EventsByCorrelation(r.Context(), workspaceUUID, correlationUUID) + events, err := h.HookService.EventsByCorrelation(r.Context(), workspaceUUID, correlationUUID, hookEventListLimit) if err != nil { h.writeHookError(w, err) return } resp := make([]DomainEventResponse, 0, len(events)) - for i, e := range events { - if i >= hookEventListLimit { - break - } + for _, e := range events { resp = append(resp, domainEventToResponse(e)) } writeJSON(w, http.StatusOK, resp) } func domainEventToResponse(e db.DomainEvent) DomainEventResponse { - return DomainEventResponse{ + // Fail-closed schema projection: expose only the event type's declared + // payload fields; drop free-text / undeclared / sensitive keys. + var payload map[string]any + if len(e.Payload) > 0 { + _ = json.Unmarshal(e.Payload, &payload) + } + projected, _ := json.Marshal(automation.ProjectPayload(e.Type, payload)) + + resp := DomainEventResponse{ ID: uuidToString(e.ID), Seq: e.Seq, Type: e.Type, + SchemaVersion: e.SchemaVersion, SubjectType: e.SubjectType, SubjectID: uuidToString(e.SubjectID), ActorType: e.ActorType, ActorID: uuidToString(e.ActorID), - Payload: rawJSON(e.Payload), + Payload: json.RawMessage(projected), CorrelationID: uuidToString(e.CorrelationID), CausationExecutionID: uuidToString(e.CausationExecutionID), HopCount: e.HopCount, CreatedAt: timestampToString(e.CreatedAt), } + if e.CausationActionIndex.Valid { + idx := e.CausationActionIndex.Int32 + resp.CausationActionIndex = &idx + } + return resp } // decodeJSONBodyStrict decodes exactly one JSON document into dst, rejecting diff --git a/server/internal/handler/hook_eval_test.go b/server/internal/handler/hook_eval_test.go index 97dfd758da9..5b4e16532bb 100644 --- a/server/internal/handler/hook_eval_test.go +++ b/server/internal/handler/hook_eval_test.go @@ -28,6 +28,22 @@ func seedStatusChangedEvent(t *testing.T, issueID, from, to, correlationID strin return id } +// seedDomainEvent inserts a domain_event with an explicit workspace, type and +// raw JSON payload. +func seedDomainEvent(t *testing.T, workspaceID, typ, subjectID, payloadJSON, correlationID string) string { + t.Helper() + var id string + if err := testPool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id) + VALUES ($1, $2, 1, 'issue', $3, 'member', $4, $5::jsonb, $6) + RETURNING id`, + workspaceID, typ, subjectID, testUserID, payloadJSON, correlationID).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + return id +} + func decodeEvaluation(t *testing.T, w *httptest.ResponseRecorder) automation.Evaluation { t.Helper() if w.Code != http.StatusOK { @@ -53,7 +69,7 @@ func TestHookDryRun(t *testing.T) { testHandler.DryRunHook(w, newMemberHookRequest(http.MethodPost, "/api/hooks/dry-run", map[string]any{"hook": sampleHookSpec("dr", "hi", issueID), "event_id": eventID})) ev := decodeEvaluation(t, w) - if !ev.Matched || !ev.WouldFire || ev.Reason != automation.ReasonMatched { + if !ev.Matched || !ev.Eligible || ev.DecisionComplete || ev.Reason != automation.ReasonMatched { t.Fatalf("expected match, got %+v", ev) } if ev.EvaluatedAgainst != automation.EvaluatedAgainstCurrentState { @@ -184,6 +200,83 @@ func TestHookEventsByCorrelation(t *testing.T) { if e.CorrelationID != correlation { t.Errorf("leaked event from correlation %s", e.CorrelationID) } + if e.SchemaVersion != 1 { + t.Errorf("schema_version = %d, want 1", e.SchemaVersion) + } + } +} + +// The correlation payload is projected to the event schema: free-text / undeclared +// keys (e.g. an issue title) are redacted, declared fields survive (review point 3). +func TestHookCorrelationRedactsPayload(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "cccccccc-cccc-cccc-cccc-cccccccccccc" + seedDomainEvent(t, testWorkspaceID, "issue.created", issueID, + `{"status":"todo","priority":"high","title":"SECRET internal title"}`, correlation) + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 1 { + t.Fatalf("events = %d, want 1", len(events)) + } + var payload map[string]any + json.Unmarshal(events[0].Payload, &payload) + if _, leaked := payload["title"]; leaked { + t.Errorf("free-text title was not redacted: %v", payload) + } + if payload["status"] != "todo" || payload["priority"] != "high" { + t.Errorf("declared payload fields were dropped: %v", payload) + } +} + +// A same-correlation event in a DIFFERENT workspace must never appear. +func TestHookCorrelationCrossWorkspaceFiltered(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + enableHooksFlag(t) + issueID := seedHookIssue(t) + const correlation = "dddddddd-dddd-dddd-dddd-dddddddddddd" + seedDomainEvent(t, testWorkspaceID, "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + // domain_event has no FK on workspace_id, so an arbitrary foreign workspace id + // with the SAME correlation must be filtered out by the workspace-scoped query. + seedDomainEvent(t, "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + + w := httptest.NewRecorder() + testHandler.ListEventsByCorrelation(w, newMemberHookRequest(http.MethodGet, "/api/events?correlation_id="+correlation, nil)) + var events []DomainEventResponse + json.NewDecoder(w.Body).Decode(&events) + if len(events) != 1 { + t.Fatalf("events = %d, want 1 (foreign-workspace same-correlation must be filtered)", len(events)) + } +} + +// The chain limit is enforced in the query, not by truncating a fully-loaded chain. +func TestHookCorrelationLimitPushedToQuery(t *testing.T) { + if testPool == nil { + t.Skip("database unavailable") + } + ctx := context.Background() + issueID := seedHookIssue(t) + const correlation = "ffffffff-ffff-ffff-ffff-ffffffffffff" + for i := 0; i < 3; i++ { + seedDomainEvent(t, testWorkspaceID, "issue.status_changed", issueID, `{"from":"todo","to":"done"}`, correlation) + } + got, err := testHandler.HookService.EventsByCorrelation(ctx, parseUUID(testWorkspaceID), parseUUID(correlation), 2) + if err != nil { + t.Fatalf("events by correlation: %v", err) + } + if len(got) != 2 { + t.Errorf("query returned %d rows, want 2 (LIMIT must be applied in SQL)", len(got)) } } diff --git a/server/internal/service/hook_eval.go b/server/internal/service/hook_eval.go index 6489ea85e17..5642e2cf4e4 100644 --- a/server/internal/service/hook_eval.go +++ b/server/internal/service/hook_eval.go @@ -107,11 +107,12 @@ func (s *HookService) Explain(ctx context.Context, workspaceID, hookID, eventID return automation.Evaluate(ctx, view, rev, &issueStateReader{q: s.Queries, workspaceID: workspaceID}) } -// EventsByCorrelation returns the domain events in a correlation chain (ordered -// by seq), for execution-chain debugging. -func (s *HookService) EventsByCorrelation(ctx context.Context, workspaceID, correlationID pgtype.UUID) ([]db.DomainEvent, error) { +// EventsByCorrelation returns up to limit domain events in a correlation chain +// (ordered by seq), for execution-chain debugging. The limit is enforced in the +// query, not by truncating a fully-loaded chain. +func (s *HookService) EventsByCorrelation(ctx context.Context, workspaceID, correlationID pgtype.UUID, limit int32) ([]db.DomainEvent, error) { return s.Queries.ListDomainEventsByCorrelation(ctx, db.ListDomainEventsByCorrelationParams{ - WorkspaceID: workspaceID, CorrelationID: correlationID, + WorkspaceID: workspaceID, CorrelationID: correlationID, Limit: limit, }) } diff --git a/server/pkg/db/generated/domain_event.sql.go b/server/pkg/db/generated/domain_event.sql.go index 7b49b12a56d..3ef79154c4f 100644 --- a/server/pkg/db/generated/domain_event.sql.go +++ b/server/pkg/db/generated/domain_event.sql.go @@ -155,15 +155,20 @@ SELECT id, seq, workspace_id, type, schema_version, subject_type, subject_id, ac WHERE workspace_id = $1 AND correlation_id = $2 ORDER BY seq ASC +LIMIT $3 ` type ListDomainEventsByCorrelationParams struct { WorkspaceID pgtype.UUID `json:"workspace_id"` CorrelationID pgtype.UUID `json:"correlation_id"` + Limit int32 `json:"limit"` } +// Bounded correlation-chain read for the debug API. The LIMIT is pushed into the +// query (not applied after loading the whole chain) and rides the +// (workspace_id, correlation_id, seq) index (MUL-4332 PR3 review round: correlation). func (q *Queries) ListDomainEventsByCorrelation(ctx context.Context, arg ListDomainEventsByCorrelationParams) ([]DomainEvent, error) { - rows, err := q.db.Query(ctx, listDomainEventsByCorrelation, arg.WorkspaceID, arg.CorrelationID) + rows, err := q.db.Query(ctx, listDomainEventsByCorrelation, arg.WorkspaceID, arg.CorrelationID, arg.Limit) if err != nil { return nil, err } diff --git a/server/pkg/db/queries/domain_event.sql b/server/pkg/db/queries/domain_event.sql index b40677b3546..68a5fc77928 100644 --- a/server/pkg/db/queries/domain_event.sql +++ b/server/pkg/db/queries/domain_event.sql @@ -30,10 +30,14 @@ SELECT * FROM domain_event WHERE id = $1; -- name: ListDomainEventsByCorrelation :many +-- Bounded correlation-chain read for the debug API. The LIMIT is pushed into the +-- query (not applied after loading the whole chain) and rides the +-- (workspace_id, correlation_id, seq) index (MUL-4332 PR3 review round: correlation). SELECT * FROM domain_event WHERE workspace_id = $1 AND correlation_id = $2 -ORDER BY seq ASC; +ORDER BY seq ASC +LIMIT $3; -- name: CountDomainEventsBySubject :one SELECT count(*) FROM domain_event From 545c23a4b14c5c0312ec9fbac43d3c9644daa55c Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Fri, 17 Jul 2026 17:59:12 +0800 Subject: [PATCH 14/15] =?UTF-8?q?feat(automation):=20durable=20matcher=20?= =?UTF-8?q?=E2=80=94=20evaluator=20+=20latch/guard/fire=20+=20snapshots=20?= =?UTF-8?q?(MUL-4332=20PR3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The durable Event Hooks matcher. It leases pending domain_event rows and, for each enabled hook whose active revision listens to that event, runs the SAME automation.Evaluate as dry-run/explain, then completes the fire decision the read-only evaluator deliberately left open, persisting one hook_execution row. No actions run — a fired hook lands `queued` for the executor (a later slice) — and the matcher loop is gated on the default-off automation_event_hooks flag, so production behaviour is unchanged. Hard acceptance (this slice): - SAME evaluator: processHookForEvent calls automation.Evaluate, never a parallel path. - Snapshot serialization: the execution's match_snapshot / condition_snapshot are ev.MatchSnapshot() / ev.ConditionSnapshot() — the exact structured result explain returns. A test asserts the stored (jsonb-parsed) snapshots deep-equal the evaluator output for the same (event, revision, state). - Final latch/guard/fire decision: the hop-depth guard (max 8), the rising-edge latch (automation_state, revision-pinned), and per_event/rising_edge fire/skip. Correctness: - Durable claim (ClaimPendingDomainEvents): lease + FOR UPDATE SKIP LOCKED, reclaims crashed matchers' expired leases → at-least-once; MarkDomainEventDispatched CASes on the lease token. - Per-hook tx locks the hook row (GetHookForUpdate) — pins the active revision against a concurrent edit AND serializes the latch read-modify-write. - Idempotent per (hook, event) via idx_hook_execution_hook_event; the latch advances exactly once (only when a new row is created), so a re-leased retry never double- fires or double-advances. - Issue scope is ownership only, not a subject filter (candidate scan filters by event_type; `when` does subject filtering). Wiring: flag-gated runHookMatcher background loop in cmd/server. New queries only, no migration change. Tests: per_event fire→queued+snapshots; condition_false→skipped; rising-edge false→true→hold(edge_not_rising)→reset→refire; hop guard; idempotency (3× match → one row, latch advanced once); snapshot-consistency (stored == evaluator output); ClaimAndMatch dispatches; disabled hook not a candidate. Next PR3 slices: the executor (effect idempotency + set_issue_status/trigger_agent), the remaining rate/concurrency/permission/fuse guards, and the crash-window tests. Co-authored-by: multica-agent --- server/cmd/server/hook_matcher.go | 41 +++ server/cmd/server/main.go | 4 + server/internal/service/hook_matcher.go | 266 ++++++++++++++ server/internal/service/hook_matcher_test.go | 329 ++++++++++++++++++ .../pkg/db/generated/automation_state.sql.go | 77 ++++ server/pkg/db/generated/domain_event.sql.go | 99 ++++++ server/pkg/db/generated/hook.sql.go | 106 ++++++ server/pkg/db/queries/automation_state.sql | 16 + server/pkg/db/queries/domain_event.sql | 29 ++ server/pkg/db/queries/hook.sql | 27 ++ 10 files changed, 994 insertions(+) create mode 100644 server/cmd/server/hook_matcher.go create mode 100644 server/internal/service/hook_matcher.go create mode 100644 server/internal/service/hook_matcher_test.go create mode 100644 server/pkg/db/generated/automation_state.sql.go create mode 100644 server/pkg/db/queries/automation_state.sql diff --git a/server/cmd/server/hook_matcher.go b/server/cmd/server/hook_matcher.go new file mode 100644 index 00000000000..a9b0cc5dc01 --- /dev/null +++ b/server/cmd/server/hook_matcher.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/multica-ai/multica/server/internal/featureflags" + "github.com/multica-ai/multica/server/internal/service" + "github.com/multica-ai/multica/server/pkg/featureflag" +) + +// hookMatcherTick is how often the matcher polls the outbox. It stays dormant +// (a cheap flag check) while automation_event_hooks is off. +const hookMatcherTick = 2 * time.Second + +// runHookMatcher is the durable Event Hooks matcher loop (MUL-4332 PR3). It only +// claims and matches pending domain events when the automation_event_hooks flag +// is enabled, so with the default-off flag it does nothing and production +// behaviour is unchanged. The matcher only records queued/skipped decisions; it +// runs no actions (the executor is a later slice). +func runHookMatcher(ctx context.Context, svc *service.HookService, flags *featureflag.Service) { + if svc == nil { + return + } + ticker := time.NewTicker(hookMatcherTick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !featureflags.EventHooksEnabled(ctx, flags) { + continue + } + if _, err := svc.ClaimAndMatch(ctx, service.MatcherBatchSize); err != nil { + slog.Warn("hook matcher tick failed", "error", err) + } + } + } +} diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index dd4678bfe99..5afabd731c2 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -389,6 +389,10 @@ func main() { // Domain event retention (MUL-4332) — explicit no-op in PR1, wired for // visibility; the real sweep lands in PR3 with hook_execution. go runDomainEventRetention(sweepCtx) + // Event Hooks matcher (MUL-4332 PR3) — consumes the domain_event outbox and + // records queued/skipped hook_execution decisions. Gated on the + // automation_event_hooks flag (default off), so it is dormant until enabled. + go runHookMatcher(sweepCtx, h.HookService, flags) go heartbeatScheduler.Run(sweepCtx) go runAutopilotFailureMonitor(autopilotCtx, queries, bus, envFailureMonitorConfig()) go runDBStatsLogger(sweepCtx, pool) diff --git a/server/internal/service/hook_matcher.go b/server/internal/service/hook_matcher.go new file mode 100644 index 00000000000..c2aacf8272c --- /dev/null +++ b/server/internal/service/hook_matcher.go @@ -0,0 +1,266 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +// The durable matcher (MUL-4332 PR3). It consumes pending domain_event rows via a +// lease, and for each enabled hook whose active revision listens to that event it +// runs the SAME automation.Evaluate as dry-run/explain, then completes the fire +// decision the read-only evaluator deliberately left open: the depth guard, the +// rising-edge latch, and per_event/rising_edge fire/skip. Each decision is +// persisted as one hook_execution row carrying the evaluator's structured +// snapshots and the pinned revision, idempotent per (hook, event). +// +// This slice does NOT run actions — a fired hook lands `queued` for the executor +// (a later slice). With no executor and the automation_event_hooks flag off +// (the matcher loop is gated on it), production behaviour is unchanged. + +const ( + hookExecQueued = "queued" + hookExecSkipped = "skipped" + + skipHopExceeded = "hop_exceeded" + skipConditionFalse = "condition_false" + skipEdgeNotRising = "edge_not_rising" + + // latchStateKind keys rising-edge latches in automation_state; one row per hook. + latchStateKind = "hook_edge" + + // maxHopCount is the loop-depth guard (§15.3): an event this deep in a + // causal chain never fires a hook. + maxHopCount = 8 + + // MatcherBatchSize / MatcherLeaseTTL bound one matcher tick. + MatcherBatchSize = 100 + MatcherLeaseTTL = 2 * time.Minute +) + +// latchState is the persisted rising-edge latch. RevisionID pins it to a +// revision so a config change starts a fresh edge rather than inheriting a stale +// satisfied flag. +type latchState struct { + Satisfied bool `json:"satisfied"` + RevisionID string `json:"revision_id"` +} + +// ClaimAndMatch leases up to batchSize pending events and matches each, marking +// only the successfully-matched ones dispatched. It returns the number +// dispatched. A per-event failure leaves that event un-dispatched so a later tick +// (after its lease expires) retries it; re-processing is idempotent. +func (s *HookService) ClaimAndMatch(ctx context.Context, batchSize int32) (int, error) { + lease := util.NewUUID() + events, err := s.Queries.ClaimPendingDomainEvents(ctx, db.ClaimPendingDomainEventsParams{ + LeaseToken: lease, + LeaseExpiresAt: pgtype.Timestamptz{Time: time.Now().Add(MatcherLeaseTTL), Valid: true}, + MaxEvents: batchSize, + }) + if err != nil { + return 0, err + } + dispatched := 0 + for _, e := range events { + if err := s.MatchEvent(ctx, e); err != nil { + slog.Warn("hook matcher: event failed, leaving for retry", "event_id", util.UUIDToString(e.ID), "error", err) + continue + } + if _, err := s.Queries.MarkDomainEventDispatched(ctx, db.MarkDomainEventDispatchedParams{ID: e.ID, LeaseToken: lease}); err != nil { + slog.Warn("hook matcher: mark dispatched failed", "event_id", util.UUIDToString(e.ID), "error", err) + continue + } + dispatched++ + } + return dispatched, nil +} + +// MatchEvent evaluates every candidate hook for one event. Each candidate is +// processed in its own transaction so a poison hook cannot block the rest. +func (s *HookService) MatchEvent(ctx context.Context, event db.DomainEvent) error { + hookIDs, err := s.Queries.ListActiveHookIDsForEvent(ctx, db.ListActiveHookIDsForEventParams{ + WorkspaceID: event.WorkspaceID, + EventType: event.Type, + }) + if err != nil { + return err + } + for _, hookID := range hookIDs { + if err := s.processHookForEvent(ctx, event, hookID); err != nil { + return err + } + } + return nil +} + +// processHookForEvent makes and persists the fire/skip decision for one (hook, +// event) pair in a single transaction. It locks the hook row first, which both +// pins the active revision against a concurrent edit and serializes the latch +// read-modify-write against other matchers. +func (s *HookService) processHookForEvent(ctx context.Context, event db.DomainEvent, hookID pgtype.UUID) error { + return s.inTx(ctx, func(qtx *db.Queries) error { + hook, err := qtx.GetHookForUpdate(ctx, db.GetHookForUpdateParams{ID: hookID, WorkspaceID: event.WorkspaceID}) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil // hook vanished between the scan and now + } + return err + } + if !hook.Enabled || hook.ArchivedAt.Valid { + return nil // disabled / archived since the scan + } + rawRev, err := qtx.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + return err + } + if rawRev.EventType != event.Type { + return nil // active revision was re-pointed to a different event type + } + + rev, err := revisionToEval(rawRev) + if err != nil { + return err + } + view, err := eventToView(event) + if err != nil { + return err + } + ev, err := automation.Evaluate(ctx, view, rev, &issueStateReader{q: qtx, workspaceID: event.WorkspaceID}) + if err != nil { + return err + } + + // A non-matching hook is not a candidate — write nothing. + if !ev.Matched { + return nil + } + + matchSnap, err := ev.MatchSnapshot() + if err != nil { + return err + } + condSnap, err := ev.ConditionSnapshot() + if err != nil { + return err + } + write := func(status, skipReason string) (bool, error) { + return writeHookExecution(ctx, qtx, event, hook, status, skipReason, matchSnap, condSnap) + } + + // Depth guard: an over-deep event never fires anything. + if event.HopCount >= maxHopCount { + _, err := write(hookExecSkipped, skipHopExceeded) + return err + } + + if rawRev.FireMode != automation.FireRisingEdge { + // per_event: fire whenever the conditions currently hold. + if ev.ConditionsMet { + _, err := write(hookExecQueued, "") + return err + } + _, err := write(hookExecSkipped, skipConditionFalse) + return err + } + + // rising_edge: fire only on a false→true transition of the latch. + prev, err := readLatch(ctx, qtx, event.WorkspaceID, hookID, hook.ActiveRevisionID) + if err != nil { + return err + } + nowSatisfied := ev.ConditionsMet + fire := nowSatisfied && !prev + + status, skipReason := hookExecQueued, "" + if !fire { + status = hookExecSkipped + if nowSatisfied { + skipReason = skipEdgeNotRising + } else { + skipReason = skipConditionFalse + } + } + inserted, err := write(status, skipReason) + if err != nil { + return err + } + // Advance the latch exactly once per (hook, event): only when THIS call + // created the execution row, so a re-leased retry never double-advances. + if inserted { + return upsertLatch(ctx, qtx, event.WorkspaceID, hookID, hook.ActiveRevisionID, nowSatisfied) + } + return nil + }) +} + +// writeHookExecution inserts one decision row idempotently. It reports whether a +// new row was created (false means it already existed — a re-processed event). +func writeHookExecution(ctx context.Context, qtx *db.Queries, event db.DomainEvent, hook db.Hook, status, skipReason string, matchSnap, condSnap []byte) (bool, error) { + reason := pgtype.Text{} + if skipReason != "" { + reason = pgtype.Text{String: skipReason, Valid: true} + } + _, err := qtx.CreateHookExecution(ctx, db.CreateHookExecutionParams{ + ID: util.NewUUID(), + WorkspaceID: event.WorkspaceID, + HookID: hook.ID, + HookRevisionID: hook.ActiveRevisionID, + EventID: event.ID, + CorrelationID: event.CorrelationID, + Status: status, + SkipReason: reason, + MatchSnapshot: matchSnap, + ConditionSnapshot: condSnap, + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil // ON CONFLICT DO NOTHING → already processed + } + return false, err + } + return true, nil +} + +// readLatch returns the previous satisfied state of a rising-edge latch, treating +// a latch pinned to a different revision as fresh (not satisfied). +func readLatch(ctx context.Context, qtx *db.Queries, workspaceID, hookID, revisionID pgtype.UUID) (bool, error) { + row, err := qtx.GetAutomationStateForUpdate(ctx, db.GetAutomationStateForUpdateParams{ + WorkspaceID: workspaceID, StateKind: latchStateKind, StateKey: util.UUIDToString(hookID), + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, err + } + var st latchState + if len(row.State) > 0 { + if err := json.Unmarshal(row.State, &st); err != nil { + return false, err + } + } + if st.RevisionID != util.UUIDToString(revisionID) { + return false, nil // stale latch from a superseded revision + } + return st.Satisfied, nil +} + +func upsertLatch(ctx context.Context, qtx *db.Queries, workspaceID, hookID, revisionID pgtype.UUID, satisfied bool) error { + state, err := json.Marshal(latchState{Satisfied: satisfied, RevisionID: util.UUIDToString(revisionID)}) + if err != nil { + return err + } + _, err = qtx.UpsertAutomationState(ctx, db.UpsertAutomationStateParams{ + WorkspaceID: workspaceID, StateKind: latchStateKind, StateKey: util.UUIDToString(hookID), State: state, + }) + return err +} diff --git a/server/internal/service/hook_matcher_test.go b/server/internal/service/hook_matcher_test.go new file mode 100644 index 00000000000..846fcb42f3d --- /dev/null +++ b/server/internal/service/hook_matcher_test.go @@ -0,0 +1,329 @@ +package service + +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/multica-ai/multica/server/internal/automation" + "github.com/multica-ai/multica/server/internal/util" + db "github.com/multica-ai/multica/server/pkg/db/generated" +) + +type matcherFixture struct { + svc *HookService + pool *pgxpool.Pool + ws string + userID string + issueID string +} + +func newMatcherFixture(t *testing.T) matcherFixture { + t.Helper() + pool := newTaskClaimRacePool(t) // skips if no DB + ws, userID, _, issueID := seedAttributionFixture(t, pool) + return matcherFixture{ + svc: NewHookService(db.New(pool), pool), + pool: pool, + ws: ws, + userID: userID, + issueID: issueID, + } +} + +// seedHook inserts a hook + immutable revision #1 directly (the matcher only +// reads them). Returns the hook id. +func (f matcherFixture) seedHook(t *testing.T, eventType, matchJSON, condJSON, fireMode string) string { + t.Helper() + ctx := context.Background() + hookID, revID := uuid.NewString(), uuid.NewString() + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook (id, workspace_id, name, enabled, active_revision_id, scope_type, origin, + creator_actor_type, creator_actor_id, authorization_principal_user_id) + VALUES ($1, $2, 'm hook', true, $3, 'workspace', 'user', 'member', $4, $4)`, + hookID, f.ws, revID, f.userID); err != nil { + t.Fatalf("seed hook: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + INSERT INTO hook_revision (id, hook_id, revision, event_type, match, conditions, fire_mode, actions, created_by_type, created_by_id) + VALUES ($1, $2, 1, $3, $4::jsonb, $5::jsonb, $6, '[]'::jsonb, 'member', $7)`, + revID, hookID, eventType, matchJSON, condJSON, fireMode, f.userID); err != nil { + t.Fatalf("seed hook_revision: %v", err) + } + t.Cleanup(func() { + bg := context.Background() + f.pool.Exec(bg, `DELETE FROM hook_execution WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM automation_state WHERE workspace_id = $1 AND state_key = $2`, f.ws, hookID) + f.pool.Exec(bg, `DELETE FROM hook_revision WHERE hook_id = $1`, hookID) + f.pool.Exec(bg, `DELETE FROM hook WHERE id = $1`, hookID) + }) + return hookID +} + +// seedEvent inserts a pending issue.status_changed domain event and returns it. +func (f matcherFixture) seedEvent(t *testing.T, to string, hopCount int) db.DomainEvent { + t.Helper() + var id string + payload := `{"from":"in_progress","to":"` + to + `"}` + if err := f.pool.QueryRow(context.Background(), ` + INSERT INTO domain_event (workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, hop_count) + VALUES ($1, 'issue.status_changed', 1, 'issue', $2, 'member', $3, $4::jsonb, gen_random_uuid(), $5) + RETURNING id`, f.ws, f.issueID, f.userID, payload, hopCount).Scan(&id); err != nil { + t.Fatalf("seed domain_event: %v", err) + } + t.Cleanup(func() { f.pool.Exec(context.Background(), `DELETE FROM domain_event WHERE id = $1`, id) }) + ev, err := f.svc.Queries.GetDomainEvent(context.Background(), util.MustParseUUID(id)) + if err != nil { + t.Fatalf("load event: %v", err) + } + return ev +} + +func (f matcherFixture) setIssueStatus(t *testing.T, status string) { + t.Helper() + if _, err := f.pool.Exec(context.Background(), `UPDATE issue SET status = $1 WHERE id = $2`, status, f.issueID); err != nil { + t.Fatalf("set issue status: %v", err) + } +} + +type execRow struct { + status string + skipReason string + revisionID string + matchSnap []byte + condSnap []byte + count int +} + +func (f matcherFixture) execFor(t *testing.T, hookID string) execRow { + t.Helper() + var r execRow + if err := f.pool.QueryRow(context.Background(), + `SELECT count(*) FROM hook_execution WHERE hook_id = $1`, hookID).Scan(&r.count); err != nil { + t.Fatalf("count executions: %v", err) + } + if r.count == 0 { + return r + } + if err := f.pool.QueryRow(context.Background(), ` + SELECT status, COALESCE(skip_reason,''), hook_revision_id::text, match_snapshot, condition_snapshot + FROM hook_execution WHERE hook_id = $1 ORDER BY created_at DESC LIMIT 1`, hookID). + Scan(&r.status, &r.skipReason, &r.revisionID, &r.matchSnap, &r.condSnap); err != nil { + t.Fatalf("load execution: %v", err) + } + return r +} + +func TestMatcherPerEventFires(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + r := f.execFor(t, hookID) + if r.count != 1 || r.status != hookExecQueued { + t.Fatalf("expected 1 queued execution, got count=%d status=%q", r.count, r.status) + } + if len(r.matchSnap) == 0 || len(r.condSnap) == 0 { + t.Errorf("execution is missing snapshots") + } +} + +func TestMatcherConditionFalseSkips(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FirePerEvent) + event := f.seedEvent(t, "in_progress", 0) + + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.status != hookExecSkipped || r.skipReason != skipConditionFalse { + t.Fatalf("expected skipped condition_false, got %+v", r) + } +} + +func TestMatcherHopGuard(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + event := f.seedEvent(t, "done", maxHopCount) // at the depth limit + + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.status != hookExecSkipped || r.skipReason != skipHopExceeded { + t.Fatalf("expected skipped hop_exceeded, got %+v", r) + } +} + +// The rising-edge latch: fires on false→true, holds while true, resets on false, +// then fires again on the next false→true. +func TestMatcherRisingEdgeLatch(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + + assertLast := func(want, reason string) { + t.Helper() + if r := f.execFor(t, hookID); r.status != want || r.skipReason != reason { + t.Fatalf("want status=%q reason=%q, got %+v", want, reason, r) + } + } + + // false→true: condition satisfied for the first time → fires. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecQueued, "") + + // still true: another matching event → no new edge. + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecSkipped, skipEdgeNotRising) + + // condition drops to false → latch resets. + f.setIssueStatus(t, "todo") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "todo", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecSkipped, skipConditionFalse) + + // false→true again → fires again. + f.setIssueStatus(t, "done") + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + assertLast(hookExecQueued, "") +} + +// Re-processing the same event must not double-create the execution or double- +// advance the latch. +func TestMatcherIdempotent(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{}`, cond, automation.FireRisingEdge) + f.setIssueStatus(t, "done") + event := f.seedEvent(t, "done", 0) + + for i := 0; i < 3; i++ { + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match %d: %v", i, err) + } + } + if r := f.execFor(t, hookID); r.count != 1 || r.status != hookExecQueued { + t.Fatalf("re-processing created %d rows (want 1), status=%q", r.count, r.status) + } + // The latch advanced exactly once: a further, distinct event must NOT re-fire. + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatal(err) + } + if r := f.execFor(t, hookID); r.status != hookExecSkipped || r.skipReason != skipEdgeNotRising { + t.Fatalf("latch double-advanced: next event fired again (%+v)", r) + } +} + +// Hard acceptance: the matcher's stored snapshots are byte-identical to the shared +// evaluator's output for the same (event, revision, state) — the same result +// explain returns, so an explanation can never drift from a real decision. +func TestMatcherSnapshotConsistency(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + f.setIssueStatus(t, "todo") + cond := `[{"issues_status":{"ids":["` + f.issueID + `"],"all":"done"}}]` + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, cond, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + // Compute the expected snapshots through the SAME evaluator path explain uses. + hook, err := f.svc.Queries.GetHookInWorkspace(ctx, db.GetHookInWorkspaceParams{ID: util.MustParseUUID(hookID), WorkspaceID: util.MustParseUUID(f.ws)}) + if err != nil { + t.Fatal(err) + } + rawRev, err := f.svc.Queries.GetHookRevision(ctx, hook.ActiveRevisionID) + if err != nil { + t.Fatal(err) + } + rev, _ := revisionToEval(rawRev) + view, _ := eventToView(event) + ev, err := automation.Evaluate(ctx, view, rev, &issueStateReader{q: f.svc.Queries, workspaceID: util.MustParseUUID(f.ws)}) + if err != nil { + t.Fatal(err) + } + if err := f.svc.MatchEvent(ctx, event); err != nil { + t.Fatalf("match: %v", err) + } + r := f.execFor(t, hookID) + + // The snapshot column is jsonb, so Postgres re-serializes it (key order / + // whitespace); compare the PARSED structured content against the evaluator's + // output — that is the drift-free contract that matters. + var gotMatch []automation.ClauseResult + if err := json.Unmarshal(r.matchSnap, &gotMatch); err != nil { + t.Fatalf("parse stored match_snapshot: %v", err) + } + if !reflect.DeepEqual(gotMatch, ev.MatchClauses) { + t.Errorf("match_snapshot content differs from evaluator:\n stored=%+v\n eval =%+v", gotMatch, ev.MatchClauses) + } + var gotCond []automation.ConditionResult + if err := json.Unmarshal(r.condSnap, &gotCond); err != nil { + t.Fatalf("parse stored condition_snapshot: %v", err) + } + if !reflect.DeepEqual(gotCond, ev.Conditions) { + t.Errorf("condition_snapshot content differs from evaluator:\n stored=%+v\n eval =%+v", gotCond, ev.Conditions) + } + if r.revisionID != util.UUIDToString(hook.ActiveRevisionID) { + t.Errorf("revision not pinned: stored=%s active=%s", r.revisionID, util.UUIDToString(hook.ActiveRevisionID)) + } +} + +// ClaimAndMatch leases a pending event, matches it, and marks it dispatched. +func TestMatcherClaimAndMatchDispatches(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + event := f.seedEvent(t, "done", 0) + + n, err := f.svc.ClaimAndMatch(ctx, MatcherBatchSize) + if err != nil { + t.Fatalf("claim and match: %v", err) + } + if n < 1 { + t.Fatalf("dispatched %d events, want >=1", n) + } + var status string + f.pool.QueryRow(ctx, `SELECT dispatch_status FROM domain_event WHERE id = $1`, util.UUIDToString(event.ID)).Scan(&status) + if status != "dispatched" { + t.Errorf("event dispatch_status = %q, want dispatched", status) + } + if r := f.execFor(t, hookID); r.status != hookExecQueued { + t.Errorf("expected queued execution after claim+match, got %+v", r) + } +} + +// A disabled hook is not a candidate. +func TestMatcherDisabledHookNotCandidate(t *testing.T) { + f := newMatcherFixture(t) + ctx := context.Background() + hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) + f.pool.Exec(ctx, `UPDATE hook SET enabled = false WHERE id = $1`, hookID) + + if err := f.svc.MatchEvent(ctx, f.seedEvent(t, "done", 0)); err != nil { + t.Fatalf("match: %v", err) + } + if r := f.execFor(t, hookID); r.count != 0 { + t.Errorf("disabled hook produced %d executions, want 0", r.count) + } +} diff --git a/server/pkg/db/generated/automation_state.sql.go b/server/pkg/db/generated/automation_state.sql.go new file mode 100644 index 00000000000..73402ff0810 --- /dev/null +++ b/server/pkg/db/generated/automation_state.sql.go @@ -0,0 +1,77 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: automation_state.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getAutomationStateForUpdate = `-- name: GetAutomationStateForUpdate :one + +SELECT workspace_id, state_kind, state_key, state, version, updated_at FROM automation_state +WHERE workspace_id = $1 AND state_kind = $2 AND state_key = $3 +FOR UPDATE +` + +type GetAutomationStateForUpdateParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` +} + +// Durable row-lockable automation state (MUL-4332): rising-edge latches (and, +// later, stage frontier + rate buckets). Not an audit log; one row per key. +// Row-lock a state key so concurrent matchers serialize their read-modify-write +// of the same latch. Returns no rows the first time a key is used. +func (q *Queries) GetAutomationStateForUpdate(ctx context.Context, arg GetAutomationStateForUpdateParams) (AutomationState, error) { + row := q.db.QueryRow(ctx, getAutomationStateForUpdate, arg.WorkspaceID, arg.StateKind, arg.StateKey) + var i AutomationState + err := row.Scan( + &i.WorkspaceID, + &i.StateKind, + &i.StateKey, + &i.State, + &i.Version, + &i.UpdatedAt, + ) + return i, err +} + +const upsertAutomationState = `-- name: UpsertAutomationState :one +INSERT INTO automation_state (workspace_id, state_kind, state_key, state, version) +VALUES ($1, $2, $3, $4, 0) +ON CONFLICT (workspace_id, state_kind, state_key) +DO UPDATE SET state = EXCLUDED.state, version = automation_state.version + 1, updated_at = now() +RETURNING workspace_id, state_kind, state_key, state, version, updated_at +` + +type UpsertAutomationStateParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + StateKind string `json:"state_kind"` + StateKey string `json:"state_key"` + State []byte `json:"state"` +} + +func (q *Queries) UpsertAutomationState(ctx context.Context, arg UpsertAutomationStateParams) (AutomationState, error) { + row := q.db.QueryRow(ctx, upsertAutomationState, + arg.WorkspaceID, + arg.StateKind, + arg.StateKey, + arg.State, + ) + var i AutomationState + err := row.Scan( + &i.WorkspaceID, + &i.StateKind, + &i.StateKey, + &i.State, + &i.Version, + &i.UpdatedAt, + ) + return i, err +} diff --git a/server/pkg/db/generated/domain_event.sql.go b/server/pkg/db/generated/domain_event.sql.go index 3ef79154c4f..82a65ca7d68 100644 --- a/server/pkg/db/generated/domain_event.sql.go +++ b/server/pkg/db/generated/domain_event.sql.go @@ -11,6 +11,84 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const claimPendingDomainEvents = `-- name: ClaimPendingDomainEvents :many + +UPDATE domain_event +SET dispatch_status = 'dispatching', + lease_token = $1, + lease_expires_at = $2, + attempts = attempts + 1 +WHERE id IN ( + SELECT id FROM domain_event + WHERE available_at <= now() + AND (dispatch_status = 'pending' + OR (dispatch_status = 'dispatching' AND lease_expires_at < now())) + ORDER BY seq ASC + LIMIT $3::int + FOR UPDATE SKIP LOCKED +) +RETURNING id, seq, workspace_id, type, schema_version, subject_type, subject_id, actor_type, actor_id, payload, correlation_id, causation_execution_id, causation_action_index, hop_count, dispatch_status, attempts, available_at, lease_token, lease_expires_at, dispatched_at, created_at +` + +type ClaimPendingDomainEventsParams struct { + LeaseToken pgtype.UUID `json:"lease_token"` + LeaseExpiresAt pgtype.Timestamptz `json:"lease_expires_at"` + MaxEvents int32 `json:"max_events"` +} + +// NOTE: the retention/TTL delete is intentionally NOT defined in PR1. The +// correct predicate is "dispatched AND older than TTL AND every related +// hook_execution is terminal" (MUL-4332 §4.1/§9), and hook_execution does not +// exist until PR3. Shipping a weaker "dispatched + TTL" delete now would risk +// reclaiming still-executing audit sources the moment PR3 enables dispatching +// (review point 5). The query lands in PR3 with the full terminal predicate. +// The durable matcher's claim scan (MUL-4332 PR3): lease a bounded batch of +// undispatched, now-available events in seq order. Reclaims events left +// 'dispatching' by a crashed matcher once their lease expires, so processing is +// at-least-once. FOR UPDATE SKIP LOCKED lets multiple matchers share the queue; +// the LIMIT bounds the lock hold. Rides idx_domain_event_dispatch. +func (q *Queries) ClaimPendingDomainEvents(ctx context.Context, arg ClaimPendingDomainEventsParams) ([]DomainEvent, error) { + rows, err := q.db.Query(ctx, claimPendingDomainEvents, arg.LeaseToken, arg.LeaseExpiresAt, arg.MaxEvents) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DomainEvent{} + for rows.Next() { + var i DomainEvent + if err := rows.Scan( + &i.ID, + &i.Seq, + &i.WorkspaceID, + &i.Type, + &i.SchemaVersion, + &i.SubjectType, + &i.SubjectID, + &i.ActorType, + &i.ActorID, + &i.Payload, + &i.CorrelationID, + &i.CausationExecutionID, + &i.CausationActionIndex, + &i.HopCount, + &i.DispatchStatus, + &i.Attempts, + &i.AvailableAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.DispatchedAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const countDomainEventsBySubject = `-- name: CountDomainEventsBySubject :one SELECT count(*) FROM domain_event WHERE subject_type = $1 @@ -208,3 +286,24 @@ func (q *Queries) ListDomainEventsByCorrelation(ctx context.Context, arg ListDom } return items, nil } + +const markDomainEventDispatched = `-- name: MarkDomainEventDispatched :execrows +UPDATE domain_event +SET dispatch_status = 'dispatched', lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 AND lease_token = $2 +` + +type MarkDomainEventDispatchedParams struct { + ID pgtype.UUID `json:"id"` + LeaseToken pgtype.UUID `json:"lease_token"` +} + +// Finalize a matched event. CAS on lease_token so only the current lease holder +// can advance it (a stale/expired matcher whose lease was reclaimed cannot). +func (q *Queries) MarkDomainEventDispatched(ctx context.Context, arg MarkDomainEventDispatchedParams) (int64, error) { + result, err := q.db.Exec(ctx, markDomainEventDispatched, arg.ID, arg.LeaseToken) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/server/pkg/db/generated/hook.sql.go b/server/pkg/db/generated/hook.sql.go index f7c604c30d4..fdeccc34838 100644 --- a/server/pkg/db/generated/hook.sql.go +++ b/server/pkg/db/generated/hook.sql.go @@ -125,6 +125,71 @@ func (q *Queries) CreateHook(ctx context.Context, arg CreateHookParams) (Hook, e return i, err } +const createHookExecution = `-- name: CreateHookExecution :one +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, skip_reason, match_snapshot, condition_snapshot +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at +` + +type CreateHookExecutionParams struct { + ID pgtype.UUID `json:"id"` + WorkspaceID pgtype.UUID `json:"workspace_id"` + HookID pgtype.UUID `json:"hook_id"` + HookRevisionID pgtype.UUID `json:"hook_revision_id"` + EventID pgtype.UUID `json:"event_id"` + CorrelationID pgtype.UUID `json:"correlation_id"` + Status string `json:"status"` + SkipReason pgtype.Text `json:"skip_reason"` + MatchSnapshot []byte `json:"match_snapshot"` + ConditionSnapshot []byte `json:"condition_snapshot"` +} + +// Persist one matcher decision (queued to fire, or skipped with a reason) with +// the evaluator's structured snapshots and the pinned revision. Idempotent per +// (hook_id, event_id) via idx_hook_execution_hook_event, so re-processing a +// re-leased event never double-creates or double-advances the latch. +func (q *Queries) CreateHookExecution(ctx context.Context, arg CreateHookExecutionParams) (HookExecution, error) { + row := q.db.QueryRow(ctx, createHookExecution, + arg.ID, + arg.WorkspaceID, + arg.HookID, + arg.HookRevisionID, + arg.EventID, + arg.CorrelationID, + arg.Status, + arg.SkipReason, + arg.MatchSnapshot, + arg.ConditionSnapshot, + ) + var i HookExecution + err := row.Scan( + &i.ID, + &i.WorkspaceID, + &i.HookID, + &i.HookRevisionID, + &i.EventID, + &i.CorrelationID, + &i.Status, + &i.SkipReason, + &i.MatchSnapshot, + &i.ConditionSnapshot, + &i.CurrentActionIndex, + &i.Attempts, + &i.NextAttemptAt, + &i.LeaseToken, + &i.LeaseExpiresAt, + &i.ErrorCode, + &i.Error, + &i.CreatedAt, + &i.StartedAt, + &i.CompletedAt, + ) + return i, err +} + const createHookRevision = `-- name: CreateHookRevision :one INSERT INTO hook_revision ( id, hook_id, revision, event_type, match, conditions, fire_mode, actions, @@ -325,6 +390,47 @@ func (q *Queries) GetMaxHookRevision(ctx context.Context, hookID pgtype.UUID) (i return max_revision, err } +const listActiveHookIDsForEvent = `-- name: ListActiveHookIDsForEvent :many +SELECT h.id +FROM hook h +JOIN hook_revision r ON r.id = h.active_revision_id +WHERE h.workspace_id = $1 + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = $2 +ORDER BY h.created_at ASC, h.id ASC +` + +type ListActiveHookIDsForEventParams struct { + WorkspaceID pgtype.UUID `json:"workspace_id"` + EventType string `json:"event_type"` +} + +// Candidate hooks for a domain event: enabled, non-archived hooks in the +// workspace whose ACTIVE revision listens to this event type. Issue scope is +// lifecycle ownership only and does NOT restrict the event subject — that is the +// job of `when` — so scope is not a filter here. The matcher re-reads each hook +// under a row lock to pin the revision authoritatively. +func (q *Queries) ListActiveHookIDsForEvent(ctx context.Context, arg ListActiveHookIDsForEventParams) ([]pgtype.UUID, error) { + rows, err := q.db.Query(ctx, listActiveHookIDsForEvent, arg.WorkspaceID, arg.EventType) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.UUID{} + for rows.Next() { + var id pgtype.UUID + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listHookExecutionsByHook = `-- name: ListHookExecutionsByHook :many SELECT id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, status, skip_reason, match_snapshot, condition_snapshot, current_action_index, attempts, next_attempt_at, lease_token, lease_expires_at, error_code, error, created_at, started_at, completed_at FROM hook_execution WHERE hook_id = $1 diff --git a/server/pkg/db/queries/automation_state.sql b/server/pkg/db/queries/automation_state.sql new file mode 100644 index 00000000000..6329f0c4ae4 --- /dev/null +++ b/server/pkg/db/queries/automation_state.sql @@ -0,0 +1,16 @@ +-- Durable row-lockable automation state (MUL-4332): rising-edge latches (and, +-- later, stage frontier + rate buckets). Not an audit log; one row per key. + +-- name: GetAutomationStateForUpdate :one +-- Row-lock a state key so concurrent matchers serialize their read-modify-write +-- of the same latch. Returns no rows the first time a key is used. +SELECT * FROM automation_state +WHERE workspace_id = $1 AND state_kind = $2 AND state_key = $3 +FOR UPDATE; + +-- name: UpsertAutomationState :one +INSERT INTO automation_state (workspace_id, state_kind, state_key, state, version) +VALUES ($1, $2, $3, $4, 0) +ON CONFLICT (workspace_id, state_kind, state_key) +DO UPDATE SET state = EXCLUDED.state, version = automation_state.version + 1, updated_at = now() +RETURNING *; diff --git a/server/pkg/db/queries/domain_event.sql b/server/pkg/db/queries/domain_event.sql index 68a5fc77928..cdca0a1db63 100644 --- a/server/pkg/db/queries/domain_event.sql +++ b/server/pkg/db/queries/domain_event.sql @@ -50,3 +50,32 @@ WHERE subject_type = $1 -- exist until PR3. Shipping a weaker "dispatched + TTL" delete now would risk -- reclaiming still-executing audit sources the moment PR3 enables dispatching -- (review point 5). The query lands in PR3 with the full terminal predicate. + +-- name: ClaimPendingDomainEvents :many +-- The durable matcher's claim scan (MUL-4332 PR3): lease a bounded batch of +-- undispatched, now-available events in seq order. Reclaims events left +-- 'dispatching' by a crashed matcher once their lease expires, so processing is +-- at-least-once. FOR UPDATE SKIP LOCKED lets multiple matchers share the queue; +-- the LIMIT bounds the lock hold. Rides idx_domain_event_dispatch. +UPDATE domain_event +SET dispatch_status = 'dispatching', + lease_token = @lease_token, + lease_expires_at = @lease_expires_at, + attempts = attempts + 1 +WHERE id IN ( + SELECT id FROM domain_event + WHERE available_at <= now() + AND (dispatch_status = 'pending' + OR (dispatch_status = 'dispatching' AND lease_expires_at < now())) + ORDER BY seq ASC + LIMIT @max_events::int + FOR UPDATE SKIP LOCKED +) +RETURNING *; + +-- name: MarkDomainEventDispatched :execrows +-- Finalize a matched event. CAS on lease_token so only the current lease holder +-- can advance it (a stale/expired matcher whose lease was reclaimed cannot). +UPDATE domain_event +SET dispatch_status = 'dispatched', lease_token = NULL, lease_expires_at = NULL +WHERE id = $1 AND lease_token = $2; diff --git a/server/pkg/db/queries/hook.sql b/server/pkg/db/queries/hook.sql index 9ddc88d0bc2..1b945e63175 100644 --- a/server/pkg/db/queries/hook.sql +++ b/server/pkg/db/queries/hook.sql @@ -101,3 +101,30 @@ SELECT * FROM hook_execution WHERE hook_id = $1 ORDER BY created_at DESC LIMIT $2; + +-- name: ListActiveHookIDsForEvent :many +-- Candidate hooks for a domain event: enabled, non-archived hooks in the +-- workspace whose ACTIVE revision listens to this event type. Issue scope is +-- lifecycle ownership only and does NOT restrict the event subject — that is the +-- job of `when` — so scope is not a filter here. The matcher re-reads each hook +-- under a row lock to pin the revision authoritatively. +SELECT h.id +FROM hook h +JOIN hook_revision r ON r.id = h.active_revision_id +WHERE h.workspace_id = $1 + AND h.enabled = true + AND h.archived_at IS NULL + AND r.event_type = $2 +ORDER BY h.created_at ASC, h.id ASC; + +-- name: CreateHookExecution :one +-- Persist one matcher decision (queued to fire, or skipped with a reason) with +-- the evaluator's structured snapshots and the pinned revision. Idempotent per +-- (hook_id, event_id) via idx_hook_execution_hook_event, so re-processing a +-- re-leased event never double-creates or double-advances the latch. +INSERT INTO hook_execution ( + id, workspace_id, hook_id, hook_revision_id, event_id, correlation_id, + status, skip_reason, match_snapshot, condition_snapshot +) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +ON CONFLICT (hook_id, event_id) DO NOTHING +RETURNING *; From c4cfdcac88ae45dcb974779d2d337459db6d236c Mon Sep 17 00:00:00 2001 From: Bohan-J Date: Fri, 17 Jul 2026 18:04:36 +0800 Subject: [PATCH 15/15] test(automation): drain to reach the matcher's event on a shared DB (MUL-4332 PR3) TestMatcherClaimAndMatchDispatches assumed one ClaimAndMatch call would reach its event, but ClaimAndMatch is a global outbox consumer and the shared CI test DB carries a backlog of undispatched events from other tests; the newest event was not in the first batch. Drain in bounded batches until the target event is dispatched (its seq is fixed and concurrently-added events have higher seq, so it converges). No production-code change. Co-authored-by: multica-agent --- server/internal/service/hook_matcher_test.go | 24 ++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/server/internal/service/hook_matcher_test.go b/server/internal/service/hook_matcher_test.go index 846fcb42f3d..f3072c1ff87 100644 --- a/server/internal/service/hook_matcher_test.go +++ b/server/internal/service/hook_matcher_test.go @@ -296,17 +296,23 @@ func TestMatcherClaimAndMatchDispatches(t *testing.T) { hookID := f.seedHook(t, "issue.status_changed", `{"to":"done"}`, `[]`, automation.FirePerEvent) event := f.seedEvent(t, "done", 0) - n, err := f.svc.ClaimAndMatch(ctx, MatcherBatchSize) - if err != nil { - t.Fatalf("claim and match: %v", err) - } - if n < 1 { - t.Fatalf("dispatched %d events, want >=1", n) - } + // ClaimAndMatch is a GLOBAL outbox consumer (not workspace-scoped) and the + // shared CI test DB carries a backlog of undispatched events from other tests. + // Our event has the newest (highest) seq, so drain in bounded batches until it + // is reached — its seq is fixed, and concurrently-added events have higher seq, + // so this converges. var status string - f.pool.QueryRow(ctx, `SELECT dispatch_status FROM domain_event WHERE id = $1`, util.UUIDToString(event.ID)).Scan(&status) + for i := 0; i < 60; i++ { + if _, err := f.svc.ClaimAndMatch(ctx, 500); err != nil { + t.Fatalf("claim and match: %v", err) + } + f.pool.QueryRow(ctx, `SELECT dispatch_status FROM domain_event WHERE id = $1`, util.UUIDToString(event.ID)).Scan(&status) + if status == "dispatched" { + break + } + } if status != "dispatched" { - t.Errorf("event dispatch_status = %q, want dispatched", status) + t.Fatalf("event dispatch_status = %q after draining, want dispatched", status) } if r := f.execFor(t, hookID); r.status != hookExecQueued { t.Errorf("expected queued execution after claim+match, got %+v", r)