From b09cdc0843a51aee842ae3ea9ddf29b1cdc495af Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 22 Dec 2025 15:00:50 -0800 Subject: [PATCH] chore: Add TODOs to investigate invalid state transition errors --- config/kubernetes/executor-job.yaml | 1 + database/boltdb/events.go | 3 +++ database/mongodb/events.go | 3 +++ tes/states.go | 1 + 4 files changed, 8 insertions(+) diff --git a/config/kubernetes/executor-job.yaml b/config/kubernetes/executor-job.yaml index 0334e3632..24e1b1a74 100644 --- a/config/kubernetes/executor-job.yaml +++ b/config/kubernetes/executor-job.yaml @@ -1,3 +1,4 @@ +# TODO: Remove all K8s artifacts in favor of Helm configs (confusing to have both) # Task Executor apiVersion: batch/v1 kind: Job diff --git a/database/boltdb/events.go b/database/boltdb/events.go index 72877d55e..01b828ec1 100644 --- a/database/boltdb/events.go +++ b/database/boltdb/events.go @@ -173,6 +173,9 @@ func transitionTaskState(tx *bolt.Tx, id string, target tes.State) error { return fmt.Errorf("Won't switch between two terminal states: %s -> %s", current, target) + // In case where restarts of executor are allowed, this line should not be encountered + // e.g. Accessing invalid object shpuld be SYSTEM_ERROR every time, + // but if a retry mechanism would work (e.g. unavailable for a few seconds), then Task Worker should allow Executor to restart as expected. case tes.TerminalState(current) && !tes.TerminalState(target): // Error when trying to switch out of a terminal state to a non-terminal one. return fmt.Errorf("Unexpected transition from %s to %s", current.String(), target.String()) diff --git a/database/mongodb/events.go b/database/mongodb/events.go index ea8bb04bb..2504e93ac 100644 --- a/database/mongodb/events.go +++ b/database/mongodb/events.go @@ -37,6 +37,9 @@ func (db *MongoDB) WriteEvent(ctx context.Context, req *events.Event) error { case events.Type_TASK_STATE: retrier := util.NewRetrier() + // TODO: This could be where the invalid state transition errors are originating from (credit @Sai) + // Check how this interacts with K8s BackoffLimit on Job spec + // Ideally we would have users be able to control their own retry policies outside of internal Funnel's logic retrier.ShouldRetry = func(err error) bool { _, isTransitionError := err.(*tes.TransitionError) return !isTransitionError && err != tes.ErrNotFound && err != tes.ErrNotPermitted diff --git a/tes/states.go b/tes/states.go index 477cf12b0..3c83d96ea 100644 --- a/tes/states.go +++ b/tes/states.go @@ -21,6 +21,7 @@ type TransitionError struct { } func (te *TransitionError) Error() string { + // TODO: This error is being thrown after valid worker restarts (e.g. K8s) return fmt.Sprintf("invalid state transition from %s to %s", te.From.String(), te.To.String()) }