diff --git a/api-contracts/v1/workflows.proto b/api-contracts/v1/workflows.proto index e155664f61..3b822e8259 100644 --- a/api-contracts/v1/workflows.proto +++ b/api-contracts/v1/workflows.proto @@ -111,8 +111,18 @@ message CreateWorkflowVersionRequest { repeated Concurrency concurrency_arr = 12; // (optional) the workflow concurrency options repeated DefaultFilter default_filters = 13; // (optional) the default filters for the workflow optional bytes input_json_schema = 14; // (optional) the JSON schema for the workflow input + + optional IdempotencyConfig idempotency = 15; // (optional) idempotency configuration for the workflow +} + +message IdempotencyConfig { + string expression = 1; // a CEL expression for determining the idempotency key for workflow runs + int64 ttl_ms = 2; // time-to-live for idempotency keys in milliseconds } +message IdempotencyCollisionError { + string existing_run_external_id = 1; // the external ID of the existing workflow run that caused the collision +} message DefaultFilter { string expression = 1; // (required) the CEL expression for the filter diff --git a/api/v1/server/oas/transformers/v1/events.go b/api/v1/server/oas/transformers/v1/events.go index e2255219d9..627f8cfdbf 100644 --- a/api/v1/server/oas/transformers/v1/events.go +++ b/api/v1/server/oas/transformers/v1/events.go @@ -77,7 +77,7 @@ func ToV1Event(event *v1.EventWithPayload) gen.V1Event { }, Payload: &payload, SeenAt: &event.EventSeenAt.Time, - Scope: &event.EventScope, + Scope: event.EventScope, TriggeredRuns: &triggeredRuns, TriggeringWebhookName: event.TriggeringWebhookName, } diff --git a/cmd/hatchet-migrate/migrate/migrations/20260708180517_v1_0_126.sql b/cmd/hatchet-migrate/migrate/migrations/20260708180517_v1_0_126.sql new file mode 100644 index 0000000000..9ba10b0ec2 --- /dev/null +++ b/cmd/hatchet-migrate/migrate/migrations/20260708180517_v1_0_126.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE "WorkflowVersion" + ADD COLUMN "idempotencyKeyExpression" TEXT, + ADD COLUMN "idempotencyKeyTtlMs" BIGINT + ; + +ALTER TYPE v1_cel_evaluation_failure_source ADD VALUE IF NOT EXISTS 'IDEMPOTENCY_KEY'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE "WorkflowVersion" + DROP COLUMN "idempotencyKeyExpression", + DROP COLUMN "idempotencyKeyTtlMs" +; +-- +goose StatementEnd diff --git a/examples/go/idempotency/trigger.go b/examples/go/idempotency/trigger.go new file mode 100644 index 0000000000..9140a21a7d --- /dev/null +++ b/examples/go/idempotency/trigger.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create hatchet client: %v", err) + } + + idempotentTask := IdempotentTask(client) + + worker, err := client.NewWorker("idempotency-worker", hatchet.WithWorkflows(idempotentTask)) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + go func() { + if err := worker.StartBlocking(interruptCtx); err != nil { + log.Fatalf("failed to start worker: %v", err) + } + }() + + ctx := context.Background() + + // > trigger + ref1, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + if err != nil { + log.Fatalf("unexpected error on first run: %v", err) + } + + ref2, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + + var runID2 string + + if err != nil { + if idempErr, ok := hatchet.IsIdempotencyCollisionError(err); ok { + fmt.Printf("Run %s already exists for this idempotency key\n", idempErr.ExistingRunExternalId) + runID2 = idempErr.ExistingRunExternalId + } else { + log.Fatalf("unexpected error on second run: %v", err) + } + } else { + runID2 = ref2.RunId + } + + fmt.Printf("First run ID: %s\n", ref1.RunId) + fmt.Printf("Second run ID: %s\n", runID2) +} diff --git a/examples/go/idempotency/trigger/main.go b/examples/go/idempotency/trigger/main.go new file mode 100644 index 0000000000..9e83811206 --- /dev/null +++ b/examples/go/idempotency/trigger/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +type IdempotencyInput struct { + ID string `json:"id"` +} + +type IdempotencyOutput struct { + Result string `json:"result"` +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create hatchet client: %v", err) + } + + idempotentTask := client.NewStandaloneTask( + "idempotent-task", + func(ctx hatchet.Context, input IdempotencyInput) (*IdempotencyOutput, error) { + return &IdempotencyOutput{ + Result: fmt.Sprintf("Hello, world from task %s", input.ID), + }, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: time.Minute, + }), + ) + + ctx := context.Background() + + // > trigger + ref1, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + if err != nil { + log.Fatalf("unexpected error on first run: %v", err) + } + + ref2, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + + var runID2 string + + if err != nil { + if idempErr, ok := hatchet.IsIdempotencyCollisionError(err); ok { + fmt.Printf("Run %s already exists for this idempotency key\n", idempErr.ExistingRunExternalId) + runID2 = idempErr.ExistingRunExternalId + } else { + log.Fatalf("unexpected error on second run: %v", err) + } + } else { + runID2 = ref2.RunId + } + + fmt.Printf("First run ID: %s\n", ref1.RunId) + fmt.Printf("Second run ID: %s\n", runID2) +} diff --git a/examples/go/idempotency/worker.go b/examples/go/idempotency/worker.go new file mode 100644 index 0000000000..4d8f4d7c61 --- /dev/null +++ b/examples/go/idempotency/worker.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + "time" + + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +type IdempotencyInput struct { + ID string `json:"id"` +} + +type IdempotencyOutput struct { + Result string `json:"result"` +} + +// > idempotency +func IdempotentTask(client *hatchet.Client) *hatchet.StandaloneTask { + return client.NewStandaloneTask( + "idempotent-task", + func(ctx hatchet.Context, input IdempotencyInput) (*IdempotencyOutput, error) { + return &IdempotencyOutput{ + Result: fmt.Sprintf("Hello, world from task %s", input.ID), + }, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: time.Minute, + }), + ) +} diff --git a/examples/go/idempotency/worker/main.go b/examples/go/idempotency/worker/main.go new file mode 100644 index 0000000000..8689509973 --- /dev/null +++ b/examples/go/idempotency/worker/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "log" + "time" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +type IdempotencyInput struct { + ID string `json:"id"` +} + +type IdempotencyOutput struct { + Result string `json:"result"` +} + +// > idempotency +func IdempotentTask(client *hatchet.Client) *hatchet.StandaloneTask { + return client.NewStandaloneTask( + "idempotent-task", + func(ctx hatchet.Context, input IdempotencyInput) (*IdempotencyOutput, error) { + return &IdempotencyOutput{ + Result: fmt.Sprintf("Hello, world from task %s", input.ID), + }, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: time.Minute, + }), + ) +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create hatchet client: %v", err) + } + + worker, err := client.NewWorker("idempotency-worker", + hatchet.WithWorkflows(IdempotentTask(client)), + ) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + if err := worker.StartBlocking(interruptCtx); err != nil { + log.Fatalf("failed to start worker: %v", err) + } +} diff --git a/examples/python/idempotency/trigger.py b/examples/python/idempotency/trigger.py new file mode 100644 index 0000000000..239fdb917f --- /dev/null +++ b/examples/python/idempotency/trigger.py @@ -0,0 +1,31 @@ +import asyncio + +from hatchet_sdk import IdempotencyCollisionError + +from examples.idempotency.worker import idempotent_task, IdempotencyInput + + +async def main() -> None: + # > trigger + ref_1 = await idempotent_task.aio_run( + input=IdempotencyInput(id="123"), + wait_for_result=False, + ) + + try: + ref_2 = await idempotent_task.aio_run( + input=IdempotencyInput(id="123"), + wait_for_result=False, + ) + run_id_2 = ref_2.workflow_run_id + except IdempotencyCollisionError as e: + print( + f"Run with external ID {e.existing_run_external_id} already exists for this idempotency key" + ) + run_id_2 = e.existing_run_external_id + + res_1 = await ref_1.aio_result() + res_2 = await idempotent_task.aio_get_result(run_id_2) + + assert res_1 == res_2 + assert ref_1.workflow_run_id == run_id_2 diff --git a/examples/python/idempotency/worker.py b/examples/python/idempotency/worker.py new file mode 100644 index 0000000000..f9f965aeb0 --- /dev/null +++ b/examples/python/idempotency/worker.py @@ -0,0 +1,47 @@ +from hatchet_sdk import Context, Hatchet, IdempotencyConfig +from datetime import timedelta +from pydantic import BaseModel + +hatchet = Hatchet() + +# > idempotency + +EVENT_KEY = "idempotency:example" + + +class IdempotencyInput(BaseModel): + id: str + + +@hatchet.task( + idempotency=IdempotencyConfig(key_expression="input.id", ttl=timedelta(minutes=1)), + input_validator=IdempotencyInput, + on_events=[EVENT_KEY], +) +async def idempotent_task(input: IdempotencyInput, ctx: Context) -> dict[str, str]: + return {"result": f"Hello, world from task {input.id}"} + + + + +@hatchet.task( + idempotency=IdempotencyConfig(key_expression="input.id", ttl=timedelta(seconds=2)), + input_validator=IdempotencyInput, + on_events=[EVENT_KEY], +) +async def idempotent_task_short_window( + input: IdempotencyInput, ctx: Context +) -> dict[str, str]: + return {"result": f"Hello, world from task {input.id}"} + + +def main() -> None: + worker = hatchet.worker( + "test-worker", + workflows=[idempotent_task], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/examples/python/worker.py b/examples/python/worker.py index 59f74cd52a..d4ced5e503 100644 --- a/examples/python/worker.py +++ b/examples/python/worker.py @@ -103,6 +103,7 @@ durable_parent_child_key_bug, child_child_key_bug, ) +from examples.idempotency.worker import idempotent_task, idempotent_task_short_window from examples.bug_tests.durable_spawn_index_collision.worker import ( durable_spawn_index_collision, spawn_index_child_a, @@ -205,6 +206,8 @@ def main() -> None: spawn_index_child_b, durable_child_key_dedup_replay, durable_spawn_many_dags, + idempotent_task, + idempotent_task_short_window, error_raising_durable_parent, error_raising_task, ], diff --git a/examples/ruby/idempotency/trigger.rb b/examples/ruby/idempotency/trigger.rb new file mode 100644 index 0000000000..a5829fce04 --- /dev/null +++ b/examples/ruby/idempotency/trigger.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'hatchet-sdk' +require_relative 'worker' + +HATCHET = Hatchet::Client.new(debug: true) unless defined?(HATCHET) + +# > trigger +first_ref = IDEMPOTENT_TASK.run_no_wait({ 'id' => '123' }) + +second_run_id = begin + second_ref = IDEMPOTENT_TASK.run_no_wait({ 'id' => '123' }) + second_ref.workflow_run_id +rescue Hatchet::IdempotencyCollisionError => e + puts "Run #{e.existing_run_external_id} already exists for this idempotency key" + e.existing_run_external_id +end + +puts "First run: #{first_ref.workflow_run_id}" +puts "Second run (or existing): #{second_run_id}" diff --git a/examples/ruby/idempotency/worker.rb b/examples/ruby/idempotency/worker.rb new file mode 100644 index 0000000000..4a990c3035 --- /dev/null +++ b/examples/ruby/idempotency/worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'hatchet-sdk' + +HATCHET = Hatchet::Client.new(debug: true) unless defined?(HATCHET) +EVENT_KEY = 'ruby-e2e:idempotency-example' + +# > idempotency +IDEMPOTENT_TASK = HATCHET.task( + name: 'ruby-e2e-idempotent-task', + idempotency: { expression: 'input.id', ttl_ms: 60_000 }, + on_events: [EVENT_KEY] +) do |input, _ctx| + { 'result' => "Hello from task #{input['id']}" } +end + +IDEMPOTENT_TASK_SHORT_WINDOW = HATCHET.task( + name: 'ruby-e2e-idempotent-task-short-window', + idempotency: { expression: 'input.id', ttl_ms: 2_000 } +) do |input, _ctx| + { 'result' => "Hello from task #{input['id']}" } +end + +def main + worker = HATCHET.worker('idempotency-worker', workflows: [IDEMPOTENT_TASK, IDEMPOTENT_TASK_SHORT_WINDOW]) + worker.start +end + +main if __FILE__ == $PROGRAM_NAME diff --git a/examples/typescript/e2e-worker.ts b/examples/typescript/e2e-worker.ts index 1cc78776ce..f7649329ab 100644 --- a/examples/typescript/e2e-worker.ts +++ b/examples/typescript/e2e-worker.ts @@ -47,6 +47,7 @@ import { durableSleep } from './durable_sleep/workflow'; import { createLoggingWorkflow } from './logger/workflow'; import { nonRetryableWorkflow } from './non_retryable/workflow'; import { failureWorkflow } from './on_failure/workflow'; +import { idempotentTask, idempotentTaskShortWindow } from './idempotency/workflow'; import { lower } from './on_event/workflow'; import { returnExceptionsTask } from './return_exceptions/workflow'; import { runDetailTestWorkflow } from './run_details/workflow'; @@ -112,6 +113,8 @@ const workflows = [ createLoggingWorkflow(hatchet), nonRetryableWorkflow, failureWorkflow, + idempotentTask, + idempotentTaskShortWindow, lower, returnExceptionsTask, runDetailTestWorkflow, diff --git a/examples/typescript/idempotency/run.ts b/examples/typescript/idempotency/run.ts new file mode 100644 index 0000000000..cd0ca66b19 --- /dev/null +++ b/examples/typescript/idempotency/run.ts @@ -0,0 +1,27 @@ +import { IdempotencyCollisionError } from '@hatchet-dev/typescript-sdk/v1'; +import { idempotentTask } from './workflow'; + +async function main() { + // > trigger + const ref1 = await idempotentTask.runNoWait({ id: '123' }); + + let runId2: string; + try { + const ref2 = await idempotentTask.runNoWait({ id: '123' }); + runId2 = await ref2.getWorkflowRunId(); + } catch (e) { + if (e instanceof IdempotencyCollisionError) { + console.log( + `Run with external ID ${e.existingRunExternalId} already exists for this idempotency key` + ); + runId2 = e.existingRunExternalId; + } else { + throw e; + } + } + + const res1 = await ref1.result(); + console.log(`Result: ${JSON.stringify(res1)}, run ID: ${runId2}`); +} + +main().catch(console.error); diff --git a/examples/typescript/idempotency/workflow.ts b/examples/typescript/idempotency/workflow.ts new file mode 100644 index 0000000000..2d13870d64 --- /dev/null +++ b/examples/typescript/idempotency/workflow.ts @@ -0,0 +1,31 @@ +import { hatchet } from '../hatchet-client'; + +export const EVENT_KEY = 'ts-e2e:idempotency-example'; + +export type IdempotencyInput = { + id: string; +}; + +// > idempotency +export const idempotentTask = hatchet.task({ + name: 'ts-e2e-idempotent-task', + idempotency: { + expression: 'input.id', + ttlMs: 60_000, + }, + onEvents: [EVENT_KEY], + fn: async (input) => { + return { result: `Hello, world from task ${input.id}` }; + }, +}); + +export const idempotentTaskShortWindow = hatchet.task({ + name: 'ts-e2e-idempotent-task-short-window', + idempotency: { + expression: 'input.id', + ttlMs: 2_000, + }, + fn: async (input) => { + return { result: `Hello, world from task ${input.id}` }; + }, +}); diff --git a/frontend/docs/pages/reference/changelog/python.mdx b/frontend/docs/pages/reference/changelog/python.mdx index 3d934bd307..6a556d53bc 100644 --- a/frontend/docs/pages/reference/changelog/python.mdx +++ b/frontend/docs/pages/reference/changelog/python.mdx @@ -1,5 +1,11 @@ {/* AUTOGENERATED — do not edit. Run `task sync-changelog` to regenerate from sdks/python/CHANGELOG.md */} +## v1.34.0 + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks, which ensures that they're only run once in a provided time window, based on a CEL expression. + ## v1.33.17 - 2026-07-07 ### Fixed diff --git a/frontend/docs/pages/reference/changelog/ruby.mdx b/frontend/docs/pages/reference/changelog/ruby.mdx index 041a020037..96ddeab82f 100644 --- a/frontend/docs/pages/reference/changelog/ruby.mdx +++ b/frontend/docs/pages/reference/changelog/ruby.mdx @@ -1,5 +1,11 @@ {/* AUTOGENERATED — do not edit. Run `task sync-changelog` to regenerate from sdks/ruby/src/CHANGELOG.md */} +## v0.4.0 - 2026-06-03 + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks via an `idempotency` option, which ensures that they're only run once in a provided time window, based on a CEL expression. Triggers that collide with an existing run raise an `IdempotencyCollisionError` containing the existing run's ID. + ## v0.3.1 - 2026-06-12 ### Fixed diff --git a/frontend/docs/pages/reference/changelog/typescript.mdx b/frontend/docs/pages/reference/changelog/typescript.mdx index d6c8ac4ee8..8f8752deb1 100644 --- a/frontend/docs/pages/reference/changelog/typescript.mdx +++ b/frontend/docs/pages/reference/changelog/typescript.mdx @@ -1,5 +1,11 @@ {/* AUTOGENERATED — do not edit. Run `task sync-changelog` to regenerate from sdks/typescript/CHANGELOG.md */} +## v1.24.0 - 2026-06-03 + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks via an `idempotency` option, which ensures that they're only run once in a provided time window, based on a CEL expression. Triggers that collide with an existing run throw an `IdempotencyCollisionError` containing the existing run's ID. + ## v1.24.3 - 2026-06-17 ### Removed diff --git a/frontend/docs/pages/v1/_meta.js b/frontend/docs/pages/v1/_meta.js index 429226f98a..8ee3b31325 100644 --- a/frontend/docs/pages/v1/_meta.js +++ b/frontend/docs/pages/v1/_meta.js @@ -39,6 +39,7 @@ export default { concurrency: "Concurrency", "rate-limits": "Rate Limits", priority: "Priority", + idempotency: "Idempotency", "--durable-workflows-section": { title: "Durable Execution", type: "separator", diff --git a/frontend/docs/pages/v1/idempotency.mdx b/frontend/docs/pages/v1/idempotency.mdx new file mode 100644 index 0000000000..b86b1afbbb --- /dev/null +++ b/frontend/docs/pages/v1/idempotency.mdx @@ -0,0 +1,54 @@ +import { Callout, Card, Cards, Steps, Tabs } from "nextra/components"; +import UniversalTabs from "@/components/UniversalTabs"; +import { Snippet } from "@/components/code"; +import { snippets } from "@/lib/generated/snippets"; + +# Idempotency + +If you need to prevent more than one run of a task from occurring within a given time window, for instance because of duplicate event sends from a webhook that can trigger duplicate runs in Hatchet, you can achieve this by adding **idempotency** configuration to your workflow or standalone task. + +## Define an Idempotency Key + +Configuring idempotency on a workflow or standalone task requires two parameters to be set: the **expression**, which will be used to create an idempotency key from the input and additional metadata of the run that's going to be triggered, and the **TTL**, which determines how long the key should live for. + + + The idempotency key expression must evaluate to a string. + + + + + + + + + + + + + + + + + +When idempotency is configured, only one run of a workflow will occur in the time window from when the first trigger comes in until the TTL expires. For instance, if you trigger a workflow at `00:00:00 UTC` (midnight) with a TTL of five minutes, and then the same workflow is triggered again with the same inputs and metadata at `00:02:00 UTC`, `00:04:00 UTC`, and `00:06:00 UTC`, only the first one (at midnight) and the final one (at `00:06:00 UTC`) will run, and then after the second run occurs, the lock on the key will be held until `00:11:00 UTC` (five minutes after the final run was triggered). + +## Handling Collisions + +When a collision occurs, the engine will reject the workflow run, and, if the run was triggered from an SDK, then the SDK will raise an exception indicating that there was an idempotency collision. This exception will contain the id of the workflow run that already existed that had claimed the key already, so you can retrieve its output if you like. + + + + + + + + + + + + + + + + +In other cases, such as triggering by events, the idempotency collision will be swallowed, and no additional runs will be created, but the event will still be ingested correctly without an error being raised. diff --git a/hack/proto/vendor/google/rpc/status.proto b/hack/proto/vendor/google/rpc/status.proto new file mode 100644 index 0000000000..dc14c9438c --- /dev/null +++ b/hack/proto/vendor/google/rpc/status.proto @@ -0,0 +1,49 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.rpc; + +import "google/protobuf/any.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; +option java_multiple_files = true; +option java_outer_classname = "StatusProto"; +option java_package = "com.google.rpc"; +option objc_class_prefix = "RPC"; + +// The `Status` type defines a logical error model that is suitable for +// different programming environments, including REST APIs and RPC APIs. It is +// used by [gRPC](https://github.com/grpc). Each `Status` message contains +// three pieces of data: error code, error message, and error details. +// +// You can find out more about this error model and how to work with it in the +// [API Design Guide](https://cloud.google.com/apis/design/errors). +message Status { + // The status code, which should be an enum value of + // [google.rpc.Code][google.rpc.Code]. + int32 code = 1; + + // A developer-facing error message, which should be in English. Any + // user-facing error message should be localized and sent in the + // [google.rpc.Status.details][google.rpc.Status.details] field, or localized + // by the client. + string message = 2; + + // A list of messages that carry the error details. There is a common set of + // message types for APIs to use. + repeated google.protobuf.Any details = 3; +} diff --git a/internal/services/admin/admin.go b/internal/services/admin/admin.go index a6afe844b3..4d16adad6c 100644 --- a/internal/services/admin/admin.go +++ b/internal/services/admin/admin.go @@ -35,8 +35,9 @@ type AdminServiceImpl struct { localDispatcher *dispatcher.DispatcherImpl l *zerolog.Logger - tw *trigger.TriggerWriter - pubBuffer *msgqueue.MQPubBuffer + tw *trigger.TriggerWriter + pubBuffer *msgqueue.MQPubBuffer + grpcTriggersEnabled bool } type AdminServiceOpt func(*AdminServiceOpts) @@ -147,15 +148,16 @@ func NewAdminService(fs ...AdminServiceOpt) (AdminService, error) { return nil, fmt.Errorf("task queue v1 is required. use WithMessageQueueV1") } - var tw *trigger.TriggerWriter - var pubBuffer *msgqueue.MQPubBuffer + pubBuffer := msgqueue.NewMQPubBuffer(opts.mqv1) + slots := 0 if opts.grpcTriggersEnabled { - pubBuffer = msgqueue.NewMQPubBuffer(opts.mqv1) - - tw = trigger.NewTriggerWriter(opts.mqv1, opts.repov1, opts.l, pubBuffer, opts.grpcTriggerSlots, opts.promGate) + slots = opts.grpcTriggerSlots } + pubBuffer = msgqueue.NewMQPubBuffer(opts.mqv1) + tw := trigger.NewTriggerWriter(opts.mqv1, opts.repov1, opts.l, pubBuffer, slots, opts.promGate) + var localScheduler *scheduler.Scheduler if opts.optimisticSchedulingEnabled && opts.localScheduler != nil { @@ -163,15 +165,16 @@ func NewAdminService(fs ...AdminServiceOpt) (AdminService, error) { } return &AdminServiceImpl{ - repov1: opts.repov1, - mqv1: opts.mqv1, - v: opts.v, - analytics: opts.analytics, - localScheduler: localScheduler, - localDispatcher: opts.localDispatcher, - l: opts.l, - tw: tw, - pubBuffer: pubBuffer, + repov1: opts.repov1, + mqv1: opts.mqv1, + v: opts.v, + analytics: opts.analytics, + localScheduler: localScheduler, + localDispatcher: opts.localDispatcher, + l: opts.l, + tw: tw, + pubBuffer: pubBuffer, + grpcTriggersEnabled: opts.grpcTriggersEnabled, }, nil } diff --git a/internal/services/admin/server_v1.go b/internal/services/admin/server_v1.go index 7a38a46eba..c44b7578b7 100644 --- a/internal/services/admin/server_v1.go +++ b/internal/services/admin/server_v1.go @@ -76,7 +76,7 @@ func (a *AdminServiceImpl) triggerWorkflowV1(ctx context.Context, req *v1contrac return nil, fmt.Errorf("could not generate external ids: %w", err) } - err = a.ingest( + idempotencyKeyCollisions, err := a.ingest( ctx, tenantId, opt, @@ -86,6 +86,22 @@ func (a *AdminServiceImpl) triggerWorkflowV1(ctx context.Context, req *v1contrac return nil, fmt.Errorf("could not trigger workflow: %w", err) } + for _, collision := range idempotencyKeyCollisions { + if collision.RequestedExternalId == opt.ExternalId { + st, stErr := status.New(codes.AlreadyExists, "idempotency key collision").WithDetails( + &v1contracts.IdempotencyCollisionError{ + ExistingRunExternalId: collision.ExistingExternalId.String(), + }, + ) + + if stErr != nil { + return nil, status.Errorf(codes.Internal, "failed to build idempotency collision error: %v", stErr) + } + + return nil, st.Err() + } + } + additionalMeta := "" if req.AdditionalMetadata != nil { additionalMeta = *req.AdditionalMetadata @@ -141,7 +157,7 @@ func (a *AdminServiceImpl) bulkTriggerWorkflowV1(ctx context.Context, req *contr return nil, fmt.Errorf("could not generate external ids: %w", err) } - err = a.ingest( + _, err = a.ingest( ctx, tenantId, opts..., @@ -232,7 +248,7 @@ func (i *AdminServiceImpl) generateExternalIds(ctx context.Context, tenantId uui return i.repov1.Triggers().PopulateExternalIdsForWorkflow(ctx, tenantId, opts) } -func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts ...*v1.WorkflowNameTriggerOpts) error { +func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts ...*v1.WorkflowNameTriggerOpts) ([]v1.IdempotencyCollision, error) { optsToSend := make([]*v1.WorkflowNameTriggerOpts, 0) for _, opt := range opts { @@ -244,7 +260,7 @@ func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts } if len(optsToSend) == 0 { - return nil + return nil, nil } if i.localScheduler != nil { @@ -254,7 +270,7 @@ func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts localWorkerIds = i.localDispatcher.GetLocalWorkerIds() } - localAssigned, schedulingErr := i.localScheduler.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) + localAssigned, idempotencyKeyCollisions, schedulingErr := i.localScheduler.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) // if we have a scheduling error, we'll fall back to normal ingestion if schedulingErr != nil { @@ -286,53 +302,66 @@ func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts // we return nil because the failed assignments would have been requeued by the local dispatcher, // and we have already written the tasks to the database - return nil + return idempotencyKeyCollisions, nil } // if there's no scheduling error, we return here because the tasks have been scheduled optimistically if schedulingErr == nil { - return nil + return idempotencyKeyCollisions, nil } - } else if i.tw != nil { - triggerErr := i.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) + } else if i.grpcTriggersEnabled { + idempotencyKeyCollisions, triggerErr := i.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) // if we fail to trigger via gRPC, we fall back to normal ingestion if triggerErr != nil && !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { i.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") } else if triggerErr == nil { - return nil + return idempotencyKeyCollisions, nil } } + if err := i.repov1.Triggers().PopulateWorkflowIdempotencyPresence(ctx, tenantId, optsToSend); err != nil { + return nil, fmt.Errorf("could not populate workflow idempotency presence: %w", err) + } + + hasIdempotencyKeys := false + for _, opt := range optsToSend { + if opt.HasIdempotencyKey { + hasIdempotencyKeys = true + break + } + } + + // if we have _any_ runs to trigger with idempotency keys, we trigger everything in the batch directly to maintain atomicity + if hasIdempotencyKeys { + idempotencyKeyCollisions, err := i.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) + if err != nil { + return nil, fmt.Errorf("could not trigger workflows: %w", err) + } + return idempotencyKeyCollisions, nil + } + verifyErr := i.repov1.Triggers().PreflightVerifyWorkflowNameOpts(ctx, tenantId, optsToSend) if verifyErr != nil { namesNotFound := &v1.ErrNamesNotFound{} if errors.As(verifyErr, &namesNotFound) { - return status.Error( - codes.InvalidArgument, - verifyErr.Error(), - ) + return nil, status.Error(codes.InvalidArgument, verifyErr.Error()) } - return fmt.Errorf("could not verify workflow name opts: %w", verifyErr) + return nil, fmt.Errorf("could not verify workflow name opts: %w", verifyErr) } - msg, err := tasktypes.TriggerTaskMessage( - tenantId, - optsToSend..., - ) + msg, err := tasktypes.TriggerTaskMessage(tenantId, optsToSend...) if err != nil { - return fmt.Errorf("could not create event task: %w", err) + return nil, fmt.Errorf("could not create event task: %w", err) } - err = i.mqv1.SendMessage(ctx, msgqueue.TASK_PROCESSING_QUEUE, msg) - - if err != nil { - return fmt.Errorf("could not add event to task queue: %w", err) + if err = i.mqv1.SendMessage(ctx, msgqueue.TASK_PROCESSING_QUEUE, msg); err != nil { + return nil, fmt.Errorf("could not add event to task queue: %w", err) } - return nil + return nil, nil } diff --git a/internal/services/admin/v1/admin.go b/internal/services/admin/v1/admin.go index baf8dc639e..b62e134592 100644 --- a/internal/services/admin/v1/admin.go +++ b/internal/services/admin/v1/admin.go @@ -34,8 +34,9 @@ type AdminServiceImpl struct { localDispatcher *dispatcher.DispatcherImpl l *zerolog.Logger - tw *trigger.TriggerWriter - pubBuffer *msgqueue.MQPubBuffer + tw *trigger.TriggerWriter + pubBuffer *msgqueue.MQPubBuffer + grpcTriggersEnabled bool } type AdminServiceOpt func(*AdminServiceOpts) @@ -149,15 +150,16 @@ func NewAdminService(fs ...AdminServiceOpt) (AdminService, error) { return nil, fmt.Errorf("task queue is required. use WithMessageQueue") } - var tw *trigger.TriggerWriter - var pubBuffer *msgqueue.MQPubBuffer + pubBuffer := msgqueue.NewMQPubBuffer(opts.mq) + slots := 0 if opts.grpcTriggersEnabled { - pubBuffer = msgqueue.NewMQPubBuffer(opts.mq) - - tw = trigger.NewTriggerWriter(opts.mq, opts.repo, opts.l, pubBuffer, opts.grpcTriggerSlots, opts.promGate) + slots = opts.grpcTriggerSlots } + pubBuffer = msgqueue.NewMQPubBuffer(opts.mq) + tw := trigger.NewTriggerWriter(opts.mq, opts.repo, opts.l, pubBuffer, slots, opts.promGate) + var localScheduler *scheduler.Scheduler if opts.optimisticSchedulingEnabled && opts.localScheduler != nil { @@ -165,15 +167,16 @@ func NewAdminService(fs ...AdminServiceOpt) (AdminService, error) { } return &AdminServiceImpl{ - repo: opts.repo, - mq: opts.mq, - v: opts.v, - analytics: opts.analytics, - localScheduler: localScheduler, - localDispatcher: opts.localDispatcher, - l: opts.l, - tw: tw, - pubBuffer: pubBuffer, + repo: opts.repo, + mq: opts.mq, + v: opts.v, + analytics: opts.analytics, + localScheduler: localScheduler, + localDispatcher: opts.localDispatcher, + l: opts.l, + tw: tw, + pubBuffer: pubBuffer, + grpcTriggersEnabled: opts.grpcTriggersEnabled, }, nil } diff --git a/internal/services/admin/v1/server.go b/internal/services/admin/v1/server.go index 6873d51c2c..3382cf8a30 100644 --- a/internal/services/admin/v1/server.go +++ b/internal/services/admin/v1/server.go @@ -15,7 +15,6 @@ import ( "github.com/hatchet-dev/hatchet/internal/listutils" "github.com/hatchet-dev/hatchet/internal/msgqueue" - "github.com/hatchet-dev/hatchet/internal/services/controllers/task/trigger" contracts "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" tasktypes "github.com/hatchet-dev/hatchet/internal/services/shared/tasktypes/v1" "github.com/hatchet-dev/hatchet/internal/statusutils" @@ -421,7 +420,7 @@ func (a *AdminServiceImpl) TriggerWorkflowRun(ctx context.Context, req *contract return nil, fmt.Errorf("could not generate external ids: %w", err) } - err = a.ingest( + idempotencyKeyCollisions, err := a.ingest( ctx, tenantId, opt, @@ -431,6 +430,22 @@ func (a *AdminServiceImpl) TriggerWorkflowRun(ctx context.Context, req *contract return nil, err } + for _, collision := range idempotencyKeyCollisions { + if collision.RequestedExternalId == opt.ExternalId { + st, err := status.New(codes.AlreadyExists, "idempotency key collision").WithDetails( + &contracts.IdempotencyCollisionError{ + ExistingRunExternalId: collision.ExistingExternalId.String(), + }, + ) + + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to build idempotency collision error: %v", err) + } + + return nil, st.Err() + } + } + return &contracts.TriggerWorkflowRunResponse{ ExternalId: opt.ExternalId.String(), }, nil @@ -587,7 +602,7 @@ func (a *AdminServiceImpl) generateExternalIds(ctx context.Context, tenantId uui return a.repo.Triggers().PopulateExternalIdsForWorkflow(ctx, tenantId, opts) } -func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts ...*v1.WorkflowNameTriggerOpts) error { +func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts ...*v1.WorkflowNameTriggerOpts) ([]v1.IdempotencyCollision, error) { optsToSend := make([]*v1.WorkflowNameTriggerOpts, 0) for _, opt := range opts { @@ -599,7 +614,7 @@ func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts } if len(optsToSend) == 0 { - return nil + return nil, nil } if a.localScheduler != nil { @@ -609,7 +624,7 @@ func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts localWorkerIds = a.localDispatcher.GetLocalWorkerIds() } - localAssigned, schedulingErr := a.localScheduler.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) + localAssigned, idempotencyKeyCollisions, schedulingErr := a.localScheduler.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) // if we have a scheduling error, we'll fall back to normal ingestion if schedulingErr != nil { @@ -641,55 +656,53 @@ func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts // we return nil because the failed assignments would have been requeued by the local dispatcher, // and we have already written the tasks to the database - return nil + return idempotencyKeyCollisions, nil } // if there's no scheduling error, we return here because the tasks have been scheduled optimistically if schedulingErr == nil { - return nil - } - } else if a.tw != nil { - triggerErr := a.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) - - // if we fail to trigger via gRPC, we fall back to normal ingestion - if triggerErr != nil && !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { - a.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") - } else if triggerErr == nil { - return nil + return idempotencyKeyCollisions, nil } } - verifyErr := a.repo.Triggers().PreflightVerifyWorkflowNameOpts(ctx, tenantId, optsToSend) - - if verifyErr != nil { + if err := a.repo.Triggers().PopulateWorkflowIdempotencyPresence(ctx, tenantId, optsToSend); err != nil { namesNotFound := &v1.ErrNamesNotFound{} - if errors.As(verifyErr, &namesNotFound) { - return status.Error( - codes.InvalidArgument, - verifyErr.Error(), - ) + if errors.As(err, &namesNotFound) { + return nil, status.Error(codes.InvalidArgument, err.Error()) } - return fmt.Errorf("could not verify workflow name opts: %w", verifyErr) + return nil, fmt.Errorf("could not populate workflow info: %w", err) } - msg, err := tasktypes.TriggerTaskMessage( - tenantId, - optsToSend..., - ) + hasIdempotencyKeys := false + for _, opt := range optsToSend { + if opt.HasIdempotencyKey { + hasIdempotencyKeys = true + break + } + } - if err != nil { - return fmt.Errorf("could not create event task: %w", err) + // if we have _any_ runs to trigger with idempotency keys, we trigger everything in the batch directly to maintain atomicity + if hasIdempotencyKeys { + idempotencyKeyCollisions, err := a.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) + if err != nil { + return nil, fmt.Errorf("could not trigger workflows: %w", err) + } + return idempotencyKeyCollisions, nil } - err = a.mq.SendMessage(ctx, msgqueue.TASK_PROCESSING_QUEUE, msg) + msg, err := tasktypes.TriggerTaskMessage(tenantId, optsToSend...) if err != nil { - return fmt.Errorf("could not add event to task queue: %w", err) + return nil, fmt.Errorf("could not create trigger message: %w", err) } - return nil + if err := a.mq.SendMessage(ctx, msgqueue.TASK_PROCESSING_QUEUE, msg); err != nil { + return nil, fmt.Errorf("could not send trigger message: %w", err) + } + + return nil, nil } func (a *AdminServiceImpl) PutWorkflow(ctx context.Context, req *contracts.CreateWorkflowVersionRequest) (*contracts.CreateWorkflowVersionResponse, error) { @@ -897,6 +910,15 @@ func getCreateWorkflowOpts(req *contracts.CreateWorkflowVersionRequest) (*v1.Cre }) } + var idempotency *v1.IdempotencyConfig + + if req.Idempotency != nil { + idempotency = &v1.IdempotencyConfig{ + Expression: req.Idempotency.Expression, + TTLMs: req.Idempotency.TtlMs, + } + } + return &v1.CreateWorkflowVersionOpts{ Name: req.Name, Concurrency: concurrency, @@ -910,6 +932,7 @@ func getCreateWorkflowOpts(req *contracts.CreateWorkflowVersionRequest) (*v1.Cre DefaultPriority: req.DefaultPriority, DefaultFilters: defaultFilters, InputJsonSchema: req.InputJsonSchema, + Idempotency: idempotency, }, nil } diff --git a/internal/services/controllers/olap/controller.go b/internal/services/controllers/olap/controller.go index 26d34a671b..b9fb0d809d 100644 --- a/internal/services/controllers/olap/controller.go +++ b/internal/services/controllers/olap/controller.go @@ -753,18 +753,12 @@ func (tc *OLAPControllerImpl) handleCreateEventTriggers(ctx context.Context, ten for _, msg := range msgs { for _, payload := range msg.Payloads { if payload.MaybeRunId != nil && payload.MaybeRunInsertedAt != nil { - var filterId uuid.UUID - - if payload.FilterId != nil { - filterId = *payload.FilterId - } - bulkCreateTriggersParams = append(bulkCreateTriggersParams, v1.EventTriggersFromExternalId{ RunID: *payload.MaybeRunId, RunInsertedAt: sqlchelpers.TimestamptzFromTime(*payload.MaybeRunInsertedAt), EventExternalId: payload.EventExternalId, EventSeenAt: sqlchelpers.TimestamptzFromTime(payload.EventSeenAt), - FilterId: filterId, + FilterId: payload.FilterId, }) } diff --git a/internal/services/controllers/task/controller.go b/internal/services/controllers/task/controller.go index 98b9fb187e..a19c4dea02 100644 --- a/internal/services/controllers/task/controller.go +++ b/internal/services/controllers/task/controller.go @@ -1100,7 +1100,8 @@ func (tc *TasksControllerImpl) handleProcessInternalEvents(ctx context.Context, // handleProcessEventTrigger is responsible for inserting tasks into the database based on event triggers. func (tc *TasksControllerImpl) handleProcessTaskTrigger(ctx context.Context, tenantId uuid.UUID, payloads [][]byte) error { - return tc.tw.TriggerFromWorkflowNames(ctx, tenantId, msgqueue.JSONConvert[v1.WorkflowNameTriggerOpts](payloads)) + _, err := tc.tw.TriggerFromWorkflowNames(ctx, tenantId, msgqueue.JSONConvert[v1.WorkflowNameTriggerOpts](payloads)) + return err } // processUserEventMatches looks for user event matches diff --git a/internal/services/controllers/task/trigger/trigger.go b/internal/services/controllers/task/trigger/trigger.go index 0df3f9bddb..0f6d877afe 100644 --- a/internal/services/controllers/task/trigger/trigger.go +++ b/internal/services/controllers/task/trigger/trigger.go @@ -111,7 +111,7 @@ func (tw *TriggerWriter) TriggerFromEvents(ctx context.Context, tenantId uuid.UU return nil } -func (tw *TriggerWriter) TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts) error { +func (tw *TriggerWriter) TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts) ([]v1.IdempotencyCollision, error) { // attempt to acquire a slot in the semaphore if tw.semaphore != nil { select { @@ -122,30 +122,36 @@ func (tw *TriggerWriter) TriggerFromWorkflowNames(ctx context.Context, tenantId }() default: // no slots available - return ErrNoTriggerSlots + return nil, ErrNoTriggerSlots } } - tasks, dags, err := tw.repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, opts) + tasks, dags, idempotencyKeyCollisions, celEvaluationFailures, err := tw.repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, opts) if err != nil { if errors.Is(err, v1.ErrResourceExhausted) { tw.l.Warn().Ctx(ctx).Str("tenantId", tenantId.String()).Msg("resource exhausted while calling TriggerFromWorkflowNames. Not retrying") - return nil + return nil, nil } - return fmt.Errorf("could not trigger workflows from names: %w", err) + return nil, fmt.Errorf("could not trigger workflows from names: %w", err) } - // signaling errors do not result in a failure, since we have already written the tasks to the database, but - // we log the error + // signaling errors do not result in a failure since we have already written the tasks to the database, + // but we log them. // FIXME: we need a mechanism to DLQ these failed signals if err := tw.signaler.SignalCreated(ctx, tenantId, tasks, dags); err != nil { tw.l.Error().Ctx(ctx).Err(err).Msg("failed to signal created tasks and DAGs in TriggerFromWorkflowNames") } - return nil + if len(celEvaluationFailures) > 0 { + if err := tw.signaler.SignalCELEvaluationFailures(ctx, tenantId, celEvaluationFailures); err != nil { + tw.l.Error().Ctx(ctx).Err(err).Msg("failed to signal CEL evaluation failures in TriggerFromWorkflowNames") + } + } + + return idempotencyKeyCollisions, nil } func (tw *TriggerWriter) SignalCreated(ctx context.Context, tenantId uuid.UUID, tasks []*v1.V1TaskWithPayload, dags []*v1.DAGWithData) error { @@ -155,3 +161,11 @@ func (tw *TriggerWriter) SignalCreated(ctx context.Context, tenantId uuid.UUID, return nil } + +func (tw *TriggerWriter) SignalCELEvaluationFailures(ctx context.Context, tenantId uuid.UUID, failures []v1.CELEvaluationFailure) error { + if err := tw.signaler.SignalCELEvaluationFailures(ctx, tenantId, failures); err != nil { + tw.l.Error().Err(err).Msg("failed to signal CEL evaluation failures in SignalCELEvaluationFailures") + } + + return nil +} diff --git a/internal/services/scheduler/v1/optimistic.go b/internal/services/scheduler/v1/optimistic.go index 102e1d0d3f..afc371d73c 100644 --- a/internal/services/scheduler/v1/optimistic.go +++ b/internal/services/scheduler/v1/optimistic.go @@ -11,11 +11,11 @@ import ( schedulingv1 "github.com/hatchet-dev/hatchet/pkg/scheduling/v1" ) -func (s *Scheduler) RunOptimisticScheduling(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts, localWorkerIds map[uuid.UUID]struct{}) (map[uuid.UUID][]*schedulingv1.AssignedItemWithTask, error) { - localTasks, tasks, dags, err := s.pool.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) +func (s *Scheduler) RunOptimisticScheduling(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts, localWorkerIds map[uuid.UUID]struct{}) (map[uuid.UUID][]*schedulingv1.AssignedItemWithTask, []v1.IdempotencyCollision, error) { + localTasks, tasks, dags, idempotencyKeyCollisions, err := s.pool.RunOptimisticScheduling(ctx, tenantId, opts, localWorkerIds) if err != nil { - return nil, err + return nil, nil, err } go func() { @@ -27,7 +27,7 @@ func (s *Scheduler) RunOptimisticScheduling(ctx context.Context, tenantId uuid.U } }() - return localTasks, err + return localTasks, idempotencyKeyCollisions, err } func (s *Scheduler) RunOptimisticSchedulingFromEvents(ctx context.Context, tenantId uuid.UUID, opts []v1.EventTriggerOpts, localWorkerIds map[uuid.UUID]struct{}) (map[uuid.UUID][]*schedulingv1.AssignedItemWithTask, error) { diff --git a/internal/services/shared/proto/v1/workflows.pb.go b/internal/services/shared/proto/v1/workflows.pb.go index db2a08f1e7..1f66621c26 100644 --- a/internal/services/shared/proto/v1/workflows.pb.go +++ b/internal/services/shared/proto/v1/workflows.pb.go @@ -789,14 +789,15 @@ type CreateWorkflowVersionRequest struct { CronTriggers []string `protobuf:"bytes,5,rep,name=cron_triggers,json=cronTriggers,proto3" json:"cron_triggers,omitempty"` // (optional) cron triggers for the workflow Tasks []*CreateTaskOpts `protobuf:"bytes,6,rep,name=tasks,proto3" json:"tasks,omitempty"` // (required) the workflow jobs // Deprecated: use concurrency_arr instead - Concurrency *Concurrency `protobuf:"bytes,7,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // (optional) the workflow concurrency options - CronInput *string `protobuf:"bytes,8,opt,name=cron_input,json=cronInput,proto3,oneof" json:"cron_input,omitempty"` // (optional) the input for the cron trigger - OnFailureTask *CreateTaskOpts `protobuf:"bytes,9,opt,name=on_failure_task,json=onFailureTask,proto3,oneof" json:"on_failure_task,omitempty"` // (optional) the job to run on failure - Sticky *StickyStrategy `protobuf:"varint,10,opt,name=sticky,proto3,enum=v1.StickyStrategy,oneof" json:"sticky,omitempty"` // (optional) the sticky strategy for assigning tasks to workers - DefaultPriority *int32 `protobuf:"varint,11,opt,name=default_priority,json=defaultPriority,proto3,oneof" json:"default_priority,omitempty"` // (optional) the default priority for the workflow - ConcurrencyArr []*Concurrency `protobuf:"bytes,12,rep,name=concurrency_arr,json=concurrencyArr,proto3" json:"concurrency_arr,omitempty"` // (optional) the workflow concurrency options - DefaultFilters []*DefaultFilter `protobuf:"bytes,13,rep,name=default_filters,json=defaultFilters,proto3" json:"default_filters,omitempty"` // (optional) the default filters for the workflow - InputJsonSchema []byte `protobuf:"bytes,14,opt,name=input_json_schema,json=inputJsonSchema,proto3,oneof" json:"input_json_schema,omitempty"` // (optional) the JSON schema for the workflow input + Concurrency *Concurrency `protobuf:"bytes,7,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // (optional) the workflow concurrency options + CronInput *string `protobuf:"bytes,8,opt,name=cron_input,json=cronInput,proto3,oneof" json:"cron_input,omitempty"` // (optional) the input for the cron trigger + OnFailureTask *CreateTaskOpts `protobuf:"bytes,9,opt,name=on_failure_task,json=onFailureTask,proto3,oneof" json:"on_failure_task,omitempty"` // (optional) the job to run on failure + Sticky *StickyStrategy `protobuf:"varint,10,opt,name=sticky,proto3,enum=v1.StickyStrategy,oneof" json:"sticky,omitempty"` // (optional) the sticky strategy for assigning tasks to workers + DefaultPriority *int32 `protobuf:"varint,11,opt,name=default_priority,json=defaultPriority,proto3,oneof" json:"default_priority,omitempty"` // (optional) the default priority for the workflow + ConcurrencyArr []*Concurrency `protobuf:"bytes,12,rep,name=concurrency_arr,json=concurrencyArr,proto3" json:"concurrency_arr,omitempty"` // (optional) the workflow concurrency options + DefaultFilters []*DefaultFilter `protobuf:"bytes,13,rep,name=default_filters,json=defaultFilters,proto3" json:"default_filters,omitempty"` // (optional) the default filters for the workflow + InputJsonSchema []byte `protobuf:"bytes,14,opt,name=input_json_schema,json=inputJsonSchema,proto3,oneof" json:"input_json_schema,omitempty"` // (optional) the JSON schema for the workflow input + Idempotency *IdempotencyConfig `protobuf:"bytes,15,opt,name=idempotency,proto3,oneof" json:"idempotency,omitempty"` // (optional) idempotency configuration for the workflow } func (x *CreateWorkflowVersionRequest) Reset() { @@ -929,6 +930,115 @@ func (x *CreateWorkflowVersionRequest) GetInputJsonSchema() []byte { return nil } +func (x *CreateWorkflowVersionRequest) GetIdempotency() *IdempotencyConfig { + if x != nil { + return x.Idempotency + } + return nil +} + +type IdempotencyConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Expression string `protobuf:"bytes,1,opt,name=expression,proto3" json:"expression,omitempty"` // a CEL expression for determining the idempotency key for workflow runs + TtlMs int64 `protobuf:"varint,2,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` // time-to-live for idempotency keys in milliseconds +} + +func (x *IdempotencyConfig) Reset() { + *x = IdempotencyConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_workflows_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IdempotencyConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IdempotencyConfig) ProtoMessage() {} + +func (x *IdempotencyConfig) ProtoReflect() protoreflect.Message { + mi := &file_v1_workflows_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IdempotencyConfig.ProtoReflect.Descriptor instead. +func (*IdempotencyConfig) Descriptor() ([]byte, []int) { + return file_v1_workflows_proto_rawDescGZIP(), []int{10} +} + +func (x *IdempotencyConfig) GetExpression() string { + if x != nil { + return x.Expression + } + return "" +} + +func (x *IdempotencyConfig) GetTtlMs() int64 { + if x != nil { + return x.TtlMs + } + return 0 +} + +type IdempotencyCollisionError struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ExistingRunExternalId string `protobuf:"bytes,1,opt,name=existing_run_external_id,json=existingRunExternalId,proto3" json:"existing_run_external_id,omitempty"` // the external ID of the existing workflow run that caused the collision +} + +func (x *IdempotencyCollisionError) Reset() { + *x = IdempotencyCollisionError{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_workflows_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IdempotencyCollisionError) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IdempotencyCollisionError) ProtoMessage() {} + +func (x *IdempotencyCollisionError) ProtoReflect() protoreflect.Message { + mi := &file_v1_workflows_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IdempotencyCollisionError.ProtoReflect.Descriptor instead. +func (*IdempotencyCollisionError) Descriptor() ([]byte, []int) { + return file_v1_workflows_proto_rawDescGZIP(), []int{11} +} + +func (x *IdempotencyCollisionError) GetExistingRunExternalId() string { + if x != nil { + return x.ExistingRunExternalId + } + return "" +} + type DefaultFilter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -942,7 +1052,7 @@ type DefaultFilter struct { func (x *DefaultFilter) Reset() { *x = DefaultFilter{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[10] + mi := &file_v1_workflows_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -955,7 +1065,7 @@ func (x *DefaultFilter) String() string { func (*DefaultFilter) ProtoMessage() {} func (x *DefaultFilter) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[10] + mi := &file_v1_workflows_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -968,7 +1078,7 @@ func (x *DefaultFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use DefaultFilter.ProtoReflect.Descriptor instead. func (*DefaultFilter) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{10} + return file_v1_workflows_proto_rawDescGZIP(), []int{12} } func (x *DefaultFilter) GetExpression() string { @@ -1005,7 +1115,7 @@ type Concurrency struct { func (x *Concurrency) Reset() { *x = Concurrency{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[11] + mi := &file_v1_workflows_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1018,7 +1128,7 @@ func (x *Concurrency) String() string { func (*Concurrency) ProtoMessage() {} func (x *Concurrency) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[11] + mi := &file_v1_workflows_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1031,7 +1141,7 @@ func (x *Concurrency) ProtoReflect() protoreflect.Message { // Deprecated: Use Concurrency.ProtoReflect.Descriptor instead. func (*Concurrency) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{11} + return file_v1_workflows_proto_rawDescGZIP(), []int{13} } func (x *Concurrency) GetExpression() string { @@ -1081,7 +1191,7 @@ type CreateTaskOpts struct { func (x *CreateTaskOpts) Reset() { *x = CreateTaskOpts{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[12] + mi := &file_v1_workflows_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1094,7 +1204,7 @@ func (x *CreateTaskOpts) String() string { func (*CreateTaskOpts) ProtoMessage() {} func (x *CreateTaskOpts) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[12] + mi := &file_v1_workflows_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1107,7 +1217,7 @@ func (x *CreateTaskOpts) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTaskOpts.ProtoReflect.Descriptor instead. func (*CreateTaskOpts) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{12} + return file_v1_workflows_proto_rawDescGZIP(), []int{14} } func (x *CreateTaskOpts) GetReadableId() string { @@ -1231,7 +1341,7 @@ type CreateTaskRateLimit struct { func (x *CreateTaskRateLimit) Reset() { *x = CreateTaskRateLimit{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[13] + mi := &file_v1_workflows_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1244,7 +1354,7 @@ func (x *CreateTaskRateLimit) String() string { func (*CreateTaskRateLimit) ProtoMessage() {} func (x *CreateTaskRateLimit) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[13] + mi := &file_v1_workflows_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1257,7 +1367,7 @@ func (x *CreateTaskRateLimit) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTaskRateLimit.ProtoReflect.Descriptor instead. func (*CreateTaskRateLimit) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{13} + return file_v1_workflows_proto_rawDescGZIP(), []int{15} } func (x *CreateTaskRateLimit) GetKey() string { @@ -1315,7 +1425,7 @@ type CreateWorkflowVersionResponse struct { func (x *CreateWorkflowVersionResponse) Reset() { *x = CreateWorkflowVersionResponse{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[14] + mi := &file_v1_workflows_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1328,7 +1438,7 @@ func (x *CreateWorkflowVersionResponse) String() string { func (*CreateWorkflowVersionResponse) ProtoMessage() {} func (x *CreateWorkflowVersionResponse) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[14] + mi := &file_v1_workflows_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1341,7 +1451,7 @@ func (x *CreateWorkflowVersionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkflowVersionResponse.ProtoReflect.Descriptor instead. func (*CreateWorkflowVersionResponse) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{14} + return file_v1_workflows_proto_rawDescGZIP(), []int{16} } func (x *CreateWorkflowVersionResponse) GetId() string { @@ -1369,7 +1479,7 @@ type GetRunDetailsRequest struct { func (x *GetRunDetailsRequest) Reset() { *x = GetRunDetailsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[15] + mi := &file_v1_workflows_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1382,7 +1492,7 @@ func (x *GetRunDetailsRequest) String() string { func (*GetRunDetailsRequest) ProtoMessage() {} func (x *GetRunDetailsRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[15] + mi := &file_v1_workflows_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1395,7 +1505,7 @@ func (x *GetRunDetailsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRunDetailsRequest.ProtoReflect.Descriptor instead. func (*GetRunDetailsRequest) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{15} + return file_v1_workflows_proto_rawDescGZIP(), []int{17} } func (x *GetRunDetailsRequest) GetExternalId() string { @@ -1421,7 +1531,7 @@ type TaskRunDetail struct { func (x *TaskRunDetail) Reset() { *x = TaskRunDetail{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[16] + mi := &file_v1_workflows_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1434,7 +1544,7 @@ func (x *TaskRunDetail) String() string { func (*TaskRunDetail) ProtoMessage() {} func (x *TaskRunDetail) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[16] + mi := &file_v1_workflows_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1447,7 +1557,7 @@ func (x *TaskRunDetail) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskRunDetail.ProtoReflect.Descriptor instead. func (*TaskRunDetail) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{16} + return file_v1_workflows_proto_rawDescGZIP(), []int{18} } func (x *TaskRunDetail) GetExternalId() string { @@ -1508,7 +1618,7 @@ type GetRunDetailsResponse struct { func (x *GetRunDetailsResponse) Reset() { *x = GetRunDetailsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_v1_workflows_proto_msgTypes[17] + mi := &file_v1_workflows_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1521,7 +1631,7 @@ func (x *GetRunDetailsResponse) String() string { func (*GetRunDetailsResponse) ProtoMessage() {} func (x *GetRunDetailsResponse) ProtoReflect() protoreflect.Message { - mi := &file_v1_workflows_proto_msgTypes[17] + mi := &file_v1_workflows_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1534,7 +1644,7 @@ func (x *GetRunDetailsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRunDetailsResponse.ProtoReflect.Descriptor instead. func (*GetRunDetailsResponse) Descriptor() ([]byte, []int) { - return file_v1_workflows_proto_rawDescGZIP(), []int{17} + return file_v1_workflows_proto_rawDescGZIP(), []int{19} } func (x *GetRunDetailsResponse) GetInput() []byte { @@ -1670,7 +1780,7 @@ var file_v1_workflows_proto_rawDesc = []byte{ 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x64, 0x22, - 0xdd, 0x05, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0xab, 0x06, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, @@ -1710,12 +1820,27 @@ var file_v1_workflows_proto_rawDesc = []byte{ 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x11, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x04, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x72, 0x6f, 0x6e, 0x5f, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, - 0x69, 0x63, 0x6b, 0x79, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, + 0x65, 0x6d, 0x61, 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a, 0x0b, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, + 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x76, 0x31, + 0x2e, 0x49, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x48, 0x05, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, + 0x79, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x72, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, + 0x72, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x69, 0x63, + 0x6b, 0x79, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, + 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x42, 0x0e, 0x0a, + 0x0c, 0x5f, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0x4a, 0x0a, + 0x11, 0x49, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x15, 0x0a, 0x06, 0x74, 0x74, 0x6c, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x74, 0x74, 0x6c, 0x4d, 0x73, 0x22, 0x54, 0x0a, 0x19, 0x49, 0x64, 0x65, + 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6c, 0x6c, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x18, 0x65, 0x78, 0x69, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x65, 0x78, 0x69, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x52, 0x75, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0x70, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, @@ -1927,7 +2052,7 @@ func file_v1_workflows_proto_rawDescGZIP() []byte { } var file_v1_workflows_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_v1_workflows_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_v1_workflows_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_v1_workflows_proto_goTypes = []interface{}{ (StickyStrategy)(0), // 0: v1.StickyStrategy (RateLimitDuration)(0), // 1: v1.RateLimitDuration @@ -1943,64 +2068,67 @@ var file_v1_workflows_proto_goTypes = []interface{}{ (*BranchDurableTaskRequest)(nil), // 11: v1.BranchDurableTaskRequest (*BranchDurableTaskResponse)(nil), // 12: v1.BranchDurableTaskResponse (*CreateWorkflowVersionRequest)(nil), // 13: v1.CreateWorkflowVersionRequest - (*DefaultFilter)(nil), // 14: v1.DefaultFilter - (*Concurrency)(nil), // 15: v1.Concurrency - (*CreateTaskOpts)(nil), // 16: v1.CreateTaskOpts - (*CreateTaskRateLimit)(nil), // 17: v1.CreateTaskRateLimit - (*CreateWorkflowVersionResponse)(nil), // 18: v1.CreateWorkflowVersionResponse - (*GetRunDetailsRequest)(nil), // 19: v1.GetRunDetailsRequest - (*TaskRunDetail)(nil), // 20: v1.TaskRunDetail - (*GetRunDetailsResponse)(nil), // 21: v1.GetRunDetailsResponse - nil, // 22: v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry - nil, // 23: v1.CreateTaskOpts.WorkerLabelsEntry - nil, // 24: v1.CreateTaskOpts.SlotRequestsEntry - nil, // 25: v1.GetRunDetailsResponse.TaskRunsEntry - (*timestamppb.Timestamp)(nil), // 26: google.protobuf.Timestamp - (*TaskConditions)(nil), // 27: v1.TaskConditions - (*DesiredWorkerLabels)(nil), // 28: v1.DesiredWorkerLabels + (*IdempotencyConfig)(nil), // 14: v1.IdempotencyConfig + (*IdempotencyCollisionError)(nil), // 15: v1.IdempotencyCollisionError + (*DefaultFilter)(nil), // 16: v1.DefaultFilter + (*Concurrency)(nil), // 17: v1.Concurrency + (*CreateTaskOpts)(nil), // 18: v1.CreateTaskOpts + (*CreateTaskRateLimit)(nil), // 19: v1.CreateTaskRateLimit + (*CreateWorkflowVersionResponse)(nil), // 20: v1.CreateWorkflowVersionResponse + (*GetRunDetailsRequest)(nil), // 21: v1.GetRunDetailsRequest + (*TaskRunDetail)(nil), // 22: v1.TaskRunDetail + (*GetRunDetailsResponse)(nil), // 23: v1.GetRunDetailsResponse + nil, // 24: v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry + nil, // 25: v1.CreateTaskOpts.WorkerLabelsEntry + nil, // 26: v1.CreateTaskOpts.SlotRequestsEntry + nil, // 27: v1.GetRunDetailsResponse.TaskRunsEntry + (*timestamppb.Timestamp)(nil), // 28: google.protobuf.Timestamp + (*TaskConditions)(nil), // 29: v1.TaskConditions + (*DesiredWorkerLabels)(nil), // 30: v1.DesiredWorkerLabels } var file_v1_workflows_proto_depIdxs = []int32{ 6, // 0: v1.CancelTasksRequest.filter:type_name -> v1.TasksFilter 6, // 1: v1.ReplayTasksRequest.filter:type_name -> v1.TasksFilter - 26, // 2: v1.TasksFilter.since:type_name -> google.protobuf.Timestamp - 26, // 3: v1.TasksFilter.until:type_name -> google.protobuf.Timestamp - 22, // 4: v1.TriggerWorkflowRunRequest.desired_worker_labels:type_name -> v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry - 16, // 5: v1.CreateWorkflowVersionRequest.tasks:type_name -> v1.CreateTaskOpts - 15, // 6: v1.CreateWorkflowVersionRequest.concurrency:type_name -> v1.Concurrency - 16, // 7: v1.CreateWorkflowVersionRequest.on_failure_task:type_name -> v1.CreateTaskOpts + 28, // 2: v1.TasksFilter.since:type_name -> google.protobuf.Timestamp + 28, // 3: v1.TasksFilter.until:type_name -> google.protobuf.Timestamp + 24, // 4: v1.TriggerWorkflowRunRequest.desired_worker_labels:type_name -> v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry + 18, // 5: v1.CreateWorkflowVersionRequest.tasks:type_name -> v1.CreateTaskOpts + 17, // 6: v1.CreateWorkflowVersionRequest.concurrency:type_name -> v1.Concurrency + 18, // 7: v1.CreateWorkflowVersionRequest.on_failure_task:type_name -> v1.CreateTaskOpts 0, // 8: v1.CreateWorkflowVersionRequest.sticky:type_name -> v1.StickyStrategy - 15, // 9: v1.CreateWorkflowVersionRequest.concurrency_arr:type_name -> v1.Concurrency - 14, // 10: v1.CreateWorkflowVersionRequest.default_filters:type_name -> v1.DefaultFilter - 3, // 11: v1.Concurrency.limit_strategy:type_name -> v1.ConcurrencyLimitStrategy - 17, // 12: v1.CreateTaskOpts.rate_limits:type_name -> v1.CreateTaskRateLimit - 23, // 13: v1.CreateTaskOpts.worker_labels:type_name -> v1.CreateTaskOpts.WorkerLabelsEntry - 15, // 14: v1.CreateTaskOpts.concurrency:type_name -> v1.Concurrency - 27, // 15: v1.CreateTaskOpts.conditions:type_name -> v1.TaskConditions - 24, // 16: v1.CreateTaskOpts.slot_requests:type_name -> v1.CreateTaskOpts.SlotRequestsEntry - 1, // 17: v1.CreateTaskRateLimit.duration:type_name -> v1.RateLimitDuration - 2, // 18: v1.TaskRunDetail.status:type_name -> v1.RunStatus - 2, // 19: v1.GetRunDetailsResponse.status:type_name -> v1.RunStatus - 25, // 20: v1.GetRunDetailsResponse.task_runs:type_name -> v1.GetRunDetailsResponse.TaskRunsEntry - 28, // 21: v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry.value:type_name -> v1.DesiredWorkerLabels - 28, // 22: v1.CreateTaskOpts.WorkerLabelsEntry.value:type_name -> v1.DesiredWorkerLabels - 20, // 23: v1.GetRunDetailsResponse.TaskRunsEntry.value:type_name -> v1.TaskRunDetail - 13, // 24: v1.AdminService.PutWorkflow:input_type -> v1.CreateWorkflowVersionRequest - 4, // 25: v1.AdminService.CancelTasks:input_type -> v1.CancelTasksRequest - 5, // 26: v1.AdminService.ReplayTasks:input_type -> v1.ReplayTasksRequest - 9, // 27: v1.AdminService.TriggerWorkflowRun:input_type -> v1.TriggerWorkflowRunRequest - 19, // 28: v1.AdminService.GetRunDetails:input_type -> v1.GetRunDetailsRequest - 11, // 29: v1.AdminService.BranchDurableTask:input_type -> v1.BranchDurableTaskRequest - 18, // 30: v1.AdminService.PutWorkflow:output_type -> v1.CreateWorkflowVersionResponse - 7, // 31: v1.AdminService.CancelTasks:output_type -> v1.CancelTasksResponse - 8, // 32: v1.AdminService.ReplayTasks:output_type -> v1.ReplayTasksResponse - 10, // 33: v1.AdminService.TriggerWorkflowRun:output_type -> v1.TriggerWorkflowRunResponse - 21, // 34: v1.AdminService.GetRunDetails:output_type -> v1.GetRunDetailsResponse - 12, // 35: v1.AdminService.BranchDurableTask:output_type -> v1.BranchDurableTaskResponse - 30, // [30:36] is the sub-list for method output_type - 24, // [24:30] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 17, // 9: v1.CreateWorkflowVersionRequest.concurrency_arr:type_name -> v1.Concurrency + 16, // 10: v1.CreateWorkflowVersionRequest.default_filters:type_name -> v1.DefaultFilter + 14, // 11: v1.CreateWorkflowVersionRequest.idempotency:type_name -> v1.IdempotencyConfig + 3, // 12: v1.Concurrency.limit_strategy:type_name -> v1.ConcurrencyLimitStrategy + 19, // 13: v1.CreateTaskOpts.rate_limits:type_name -> v1.CreateTaskRateLimit + 25, // 14: v1.CreateTaskOpts.worker_labels:type_name -> v1.CreateTaskOpts.WorkerLabelsEntry + 17, // 15: v1.CreateTaskOpts.concurrency:type_name -> v1.Concurrency + 29, // 16: v1.CreateTaskOpts.conditions:type_name -> v1.TaskConditions + 26, // 17: v1.CreateTaskOpts.slot_requests:type_name -> v1.CreateTaskOpts.SlotRequestsEntry + 1, // 18: v1.CreateTaskRateLimit.duration:type_name -> v1.RateLimitDuration + 2, // 19: v1.TaskRunDetail.status:type_name -> v1.RunStatus + 2, // 20: v1.GetRunDetailsResponse.status:type_name -> v1.RunStatus + 27, // 21: v1.GetRunDetailsResponse.task_runs:type_name -> v1.GetRunDetailsResponse.TaskRunsEntry + 30, // 22: v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry.value:type_name -> v1.DesiredWorkerLabels + 30, // 23: v1.CreateTaskOpts.WorkerLabelsEntry.value:type_name -> v1.DesiredWorkerLabels + 22, // 24: v1.GetRunDetailsResponse.TaskRunsEntry.value:type_name -> v1.TaskRunDetail + 13, // 25: v1.AdminService.PutWorkflow:input_type -> v1.CreateWorkflowVersionRequest + 4, // 26: v1.AdminService.CancelTasks:input_type -> v1.CancelTasksRequest + 5, // 27: v1.AdminService.ReplayTasks:input_type -> v1.ReplayTasksRequest + 9, // 28: v1.AdminService.TriggerWorkflowRun:input_type -> v1.TriggerWorkflowRunRequest + 21, // 29: v1.AdminService.GetRunDetails:input_type -> v1.GetRunDetailsRequest + 11, // 30: v1.AdminService.BranchDurableTask:input_type -> v1.BranchDurableTaskRequest + 20, // 31: v1.AdminService.PutWorkflow:output_type -> v1.CreateWorkflowVersionResponse + 7, // 32: v1.AdminService.CancelTasks:output_type -> v1.CancelTasksResponse + 8, // 33: v1.AdminService.ReplayTasks:output_type -> v1.ReplayTasksResponse + 10, // 34: v1.AdminService.TriggerWorkflowRun:output_type -> v1.TriggerWorkflowRunResponse + 23, // 35: v1.AdminService.GetRunDetails:output_type -> v1.GetRunDetailsResponse + 12, // 36: v1.AdminService.BranchDurableTask:output_type -> v1.BranchDurableTaskResponse + 31, // [31:37] is the sub-list for method output_type + 25, // [25:31] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_v1_workflows_proto_init() } @@ -2132,7 +2260,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DefaultFilter); i { + switch v := v.(*IdempotencyConfig); i { case 0: return &v.state case 1: @@ -2144,7 +2272,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Concurrency); i { + switch v := v.(*IdempotencyCollisionError); i { case 0: return &v.state case 1: @@ -2156,7 +2284,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTaskOpts); i { + switch v := v.(*DefaultFilter); i { case 0: return &v.state case 1: @@ -2168,7 +2296,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTaskRateLimit); i { + switch v := v.(*Concurrency); i { case 0: return &v.state case 1: @@ -2180,7 +2308,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateWorkflowVersionResponse); i { + switch v := v.(*CreateTaskOpts); i { case 0: return &v.state case 1: @@ -2192,7 +2320,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetRunDetailsRequest); i { + switch v := v.(*CreateTaskRateLimit); i { case 0: return &v.state case 1: @@ -2204,7 +2332,7 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TaskRunDetail); i { + switch v := v.(*CreateWorkflowVersionResponse); i { case 0: return &v.state case 1: @@ -2216,6 +2344,30 @@ func file_v1_workflows_proto_init() { } } file_v1_workflows_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetRunDetailsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v1_workflows_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TaskRunDetail); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v1_workflows_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetRunDetailsResponse); i { case 0: return &v.state @@ -2233,18 +2385,18 @@ func file_v1_workflows_proto_init() { file_v1_workflows_proto_msgTypes[2].OneofWrappers = []interface{}{} file_v1_workflows_proto_msgTypes[5].OneofWrappers = []interface{}{} file_v1_workflows_proto_msgTypes[9].OneofWrappers = []interface{}{} - file_v1_workflows_proto_msgTypes[10].OneofWrappers = []interface{}{} - file_v1_workflows_proto_msgTypes[11].OneofWrappers = []interface{}{} file_v1_workflows_proto_msgTypes[12].OneofWrappers = []interface{}{} file_v1_workflows_proto_msgTypes[13].OneofWrappers = []interface{}{} - file_v1_workflows_proto_msgTypes[16].OneofWrappers = []interface{}{} + file_v1_workflows_proto_msgTypes[14].OneofWrappers = []interface{}{} + file_v1_workflows_proto_msgTypes[15].OneofWrappers = []interface{}{} + file_v1_workflows_proto_msgTypes[18].OneofWrappers = []interface{}{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_workflows_proto_rawDesc, NumEnums: 4, - NumMessages: 22, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/services/ticker/schedule_workflow_v1.go b/internal/services/ticker/schedule_workflow_v1.go index 2b0ff0a61e..2cfda3d528 100644 --- a/internal/services/ticker/schedule_workflow_v1.go +++ b/internal/services/ticker/schedule_workflow_v1.go @@ -2,19 +2,15 @@ package ticker import ( "context" - "errors" "fmt" "time" "github.com/google/uuid" - "github.com/jackc/pgerrcode" - "github.com/jackc/pgx/v5/pgconn" "github.com/rs/zerolog" "github.com/hatchet-dev/hatchet/internal/msgqueue" tasktypes "github.com/hatchet-dev/hatchet/internal/services/shared/tasktypes/v1" v1 "github.com/hatchet-dev/hatchet/pkg/repository" - "github.com/hatchet-dev/hatchet/pkg/repository/sqlchelpers" ) func (t *TickerImpl) RunScheduledWorkflowV1(ctx context.Context, tenantId uuid.UUID, opts v1.RunScheduledWorkflowV1Opts) error { @@ -23,20 +19,17 @@ func (t *TickerImpl) RunScheduledWorkflowV1(ctx context.Context, tenantId uuid.U } func RunScheduledWorkflow(ctx context.Context, l *zerolog.Logger, mq msgqueue.MessageQueue, repo v1.Repository, tenantId uuid.UUID, opts v1.RunScheduledWorkflowV1Opts) (*uuid.UUID, error) { - expiresAt := opts.TriggerAt.Add(time.Second * 30) - err := repo.Idempotency().CreateIdempotencyKey(ctx, tenantId, opts.ID.String(), sqlchelpers.TimestamptzFromTime(expiresAt)) + claimed, err := repo.Idempotency().ClaimKey(ctx, tenantId, fmt.Sprintf("hatchet_internal_%s", opts.ID.String()), opts.TriggerAt.Add(30*time.Second), opts.ID) - var pgErr *pgconn.PgError - // if we get a unique violation, it means we tried to create a duplicate idempotency key, which means this - // run has already been processed, so we should just return - if err != nil && errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation { - l.Info().Ctx(ctx).Msgf("idempotency key for scheduled workflow %s already exists, skipping", opts.ID.String()) + if err != nil { + return nil, fmt.Errorf("could not claim idempotency key for scheduled workflow: %w", err) + } + + if !claimed { + l.Info().Ctx(ctx).Msgf("idempotency key for scheduled workflow %s already claimed, skipping", opts.ID.String()) return nil, nil - } else if err != nil { - return nil, fmt.Errorf("could not create idempotency key: %w", err) } - key := v1.IdempotencyKey(opts.ID.String()) externalId := uuid.New() msg, err := tasktypes.TriggerTaskMessage( @@ -48,9 +41,8 @@ func RunScheduledWorkflow(ctx context.Context, l *zerolog.Logger, mq msgqueue.Me AdditionalMetadata: opts.AdditionalMetadata, Priority: opts.Priority, }, - IdempotencyKey: &key, - ExternalId: externalId, - ShouldSkip: false, + ExternalId: externalId, + ShouldSkip: false, }, ) diff --git a/pkg/client/admin.go b/pkg/client/admin.go index 20c9edcb17..34a9e86630 100644 --- a/pkg/client/admin.go +++ b/pkg/client/admin.go @@ -174,6 +174,14 @@ func (d *DedupeViolationErr) Error() string { return fmt.Sprintf("DedupeViolationErr: %s", d.details) } +type IdempotencyViolationErr struct { + ExistingRunExternalId string +} + +func (e *IdempotencyViolationErr) Error() string { + return fmt.Sprintf("idempotency key collision: existing run %s", e.ExistingRunExternalId) +} + type adminClientImpl struct { client admincontracts.WorkflowServiceClient v1Client v1contracts.AdminServiceClient @@ -421,6 +429,15 @@ func (a *adminClientImpl) RunWorkflow(workflowName string, input interface{}, op if err != nil { if status.Code(err) == codes.AlreadyExists { + if st, ok := status.FromError(err); ok { + for _, detail := range st.Details() { + if idempotencyErr, ok := detail.(*v1contracts.IdempotencyCollisionError); ok { + return nil, &IdempotencyViolationErr{ + ExistingRunExternalId: idempotencyErr.GetExistingRunExternalId(), + } + } + } + } return nil, &DedupeViolationErr{ details: fmt.Sprintf("could not trigger workflow: %s", err.Error()), } @@ -491,6 +508,15 @@ func (a *adminClientImpl) RunChildWorkflow(workflowName string, input interface{ if err != nil { if status.Code(err) == codes.AlreadyExists { + if st, ok := status.FromError(err); ok { + for _, detail := range st.Details() { + if idempotencyErr, ok := detail.(*v1contracts.IdempotencyCollisionError); ok { + return "", &IdempotencyViolationErr{ + ExistingRunExternalId: idempotencyErr.GetExistingRunExternalId(), + } + } + } + } return "", &DedupeViolationErr{ details: fmt.Sprintf("could not trigger child workflow: %s", err.Error()), } diff --git a/pkg/client/create/tasks.go b/pkg/client/create/tasks.go index de6bf05e31..558dbaa850 100644 --- a/pkg/client/create/tasks.go +++ b/pkg/client/create/tasks.go @@ -8,6 +8,15 @@ import ( "github.com/hatchet-dev/hatchet/pkg/client/types" ) +// IdempotencyConfig configures idempotency behavior for a workflow. +type IdempotencyConfig struct { + // Expression is a CEL expression evaluated against the workflow input to produce an idempotency key. + Expression string + + // TTL is the duration during which duplicate runs with the same key are rejected. + TTL time.Duration +} + // TaskDefaults defines default configuration values for tasks within a workflow. type TaskDefaults struct { // (optional) ExecutionTimeout specifies the maximum duration a task can run after starting before being terminated @@ -62,4 +71,7 @@ type WorkflowCreateOpts[I any] struct { DefaultPriority *int32 DefaultFilters []types.DefaultFilter + + // (optional) Idempotency configuration for preventing duplicate runs + Idempotency *IdempotencyConfig } diff --git a/pkg/repository/durable_events.go b/pkg/repository/durable_events.go index 20d6cdbc0e..1c67acdc7f 100644 --- a/pkg/repository/durable_events.go +++ b/pkg/repository/durable_events.go @@ -87,10 +87,11 @@ type IngestTriggerRunsEntry struct { } type IngestTriggerRunsResult struct { - Entries []*IngestTriggerRunsEntry - CreatedTasks []*V1TaskWithPayload - CreatedDAGs []*DAGWithData - InvocationCount int32 + Entries []*IngestTriggerRunsEntry + CreatedTasks []*V1TaskWithPayload + CreatedDAGs []*DAGWithData + InvocationCount int32 + CELEvaluationFailures []CELEvaluationFailure } type IngestWaitForResult struct { @@ -1296,7 +1297,7 @@ func (r *durableEventsRepository) IngestDurableTaskEvent(ctx context.Context, op } if len(newTriggerOpts) > 0 { - createdTasks, createdDags, triggerErr := r.triggerFromWorkflowNames(ctx, optTx, tenantId, newTriggerOpts) + createdTasks, createdDags, _, celFailures, triggerErr := r.triggerFromWorkflowNames(ctx, optTx, tenantId, newTriggerOpts) if triggerErr != nil { return nil, fmt.Errorf("failed to trigger workflows: %w", triggerErr) @@ -1304,6 +1305,7 @@ func (r *durableEventsRepository) IngestDurableTaskEvent(ctx context.Context, op triggerRunsResult.CreatedTasks = createdTasks triggerRunsResult.CreatedDAGs = createdDags + triggerRunsResult.CELEvaluationFailures = celFailures createMatchOpts := make([]CreateMatchOpts, 0, len(createdTasks)+len(createdDags)) diff --git a/pkg/repository/idempotency.go b/pkg/repository/idempotency.go index 6323986037..b0045b295e 100644 --- a/pkg/repository/idempotency.go +++ b/pkg/repository/idempotency.go @@ -2,19 +2,28 @@ package repository import ( "context" + "fmt" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlchelpers" "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" ) type WasSuccessfullyClaimed bool type IdempotencyKey string +type ClaimIdempotencyKeysOpt struct { + Key string + ClaimedByExternalId uuid.UUID + ExpiresAt pgtype.Timestamptz +} + type IdempotencyRepository interface { - CreateIdempotencyKey(context context.Context, tenantId uuid.UUID, key string, expiresAt pgtype.Timestamptz) error EvictExpiredIdempotencyKeys(context context.Context, tenantId uuid.UUID) error + ClaimKey(ctx context.Context, tenantId uuid.UUID, key string, expiresAt time.Time, claimedByExternalId uuid.UUID) (bool, error) } type idempotencyRepository struct { @@ -27,55 +36,34 @@ func newIdempotencyRepository(shared *sharedRepository) IdempotencyRepository { } } -func (r *idempotencyRepository) CreateIdempotencyKey(context context.Context, tenantId uuid.UUID, key string, expiresAt pgtype.Timestamptz) error { - return r.queries.CreateIdempotencyKey(context, r.pool, sqlcv1.CreateIdempotencyKeyParams{ - Tenantid: tenantId, - Key: key, - Expiresat: expiresAt, - }) -} - func (r *idempotencyRepository) EvictExpiredIdempotencyKeys(context context.Context, tenantId uuid.UUID) error { return r.queries.CleanUpExpiredIdempotencyKeys(context, r.pool, tenantId) } -type KeyClaimantPair struct { - IdempotencyKey IdempotencyKey - ClaimedByExternalId uuid.UUID -} - -func claimIdempotencyKeys(context context.Context, queries *sqlcv1.Queries, tx sqlcv1.DBTX, tenantId uuid.UUID, claims []KeyClaimantPair) (map[KeyClaimantPair]WasSuccessfullyClaimed, error) { - keys := make([]string, len(claims)) - claimedByExternalIds := make([]uuid.UUID, len(claims)) - - for i, claim := range claims { - keys[i] = string(claim.IdempotencyKey) - claimedByExternalIds[i] = claim.ClaimedByExternalId - } - - claimResults, err := queries.ClaimIdempotencyKeys(context, tx, sqlcv1.ClaimIdempotencyKeysParams{ +func (r *idempotencyRepository) ClaimKey(ctx context.Context, tenantId uuid.UUID, key string, expiresAt time.Time, claimedByExternalId uuid.UUID) (bool, error) { + results, err := r.queries.ClaimIdempotencyKeys(ctx, r.pool, sqlcv1.ClaimIdempotencyKeysParams{ + Keys: []string{key}, + Expiresats: []pgtype.Timestamptz{sqlchelpers.TimestamptzFromTime(expiresAt)}, + Claimedbyexternalids: []uuid.UUID{claimedByExternalId}, Tenantid: tenantId, - Keys: keys, - Claimedbyexternalids: claimedByExternalIds, }) if err != nil { - return nil, err + return false, err } - keyToClaimStatus := make(map[KeyClaimantPair]WasSuccessfullyClaimed) - - for _, claimResult := range claimResults { - if claimResult.ClaimedByExternalID == nil { - continue - } + if len(results) == 0 { + return false, nil + } - keyClaimantPair := KeyClaimantPair{ - IdempotencyKey: IdempotencyKey(claimResult.Key), - ClaimedByExternalId: *claimResult.ClaimedByExternalID, - } - keyToClaimStatus[keyClaimantPair] = WasSuccessfullyClaimed(claimResult.WasSuccessfullyClaimed) + if len(results) > 1 { + return false, fmt.Errorf("unexpectedly claimed more than one idempotency key, this should never happen") } - return keyToClaimStatus, nil + return results[0].WasSuccessfullyClaimed, nil +} + +type IdempotencyCollision struct { + RequestedExternalId uuid.UUID + ExistingExternalId uuid.UUID } diff --git a/pkg/repository/ids.go b/pkg/repository/ids.go index c4dddd3103..c081b5e77f 100644 --- a/pkg/repository/ids.go +++ b/pkg/repository/ids.go @@ -27,11 +27,11 @@ type WorkflowNameTriggerOpts struct { ExternalId uuid.UUID - // (optional) The idempotency key to use for debouncing this task - IdempotencyKey *IdempotencyKey - // Whether to skip the creation of the child workflow ShouldSkip bool + + // Whether the workflow has an idempotency key expression configured. + HasIdempotencyKey bool } func (g *WorkflowNameTriggerOpts) childSpawnKey() string { diff --git a/pkg/repository/olap.go b/pkg/repository/olap.go index a7edff63ea..f9919f71c2 100644 --- a/pkg/repository/olap.go +++ b/pkg/repository/olap.go @@ -2551,7 +2551,7 @@ type EventTriggersFromExternalId struct { RunInsertedAt pgtype.Timestamptz `json:"run_inserted_at"` EventExternalId uuid.UUID `json:"event_external_id"` EventSeenAt pgtype.Timestamptz `json:"event_seen_at"` - FilterId uuid.UUID `json:"filter_id"` + FilterId *uuid.UUID `json:"filter_id"` } func (r *OLAPRepositoryImpl) BulkCreateEventsAndTriggers(ctx context.Context, events sqlcv1.BulkCreateEventsOLAPParams, triggers []EventTriggersFromExternalId) error { @@ -2607,7 +2607,7 @@ func (r *OLAPRepositoryImpl) BulkCreateEventsAndTriggers(ctx context.Context, ev RunInsertedAt: trigger.RunInsertedAt, EventID: eventId, EventSeenAt: trigger.EventSeenAt, - FilterID: &trigger.FilterId, + FilterID: trigger.FilterId, }) } @@ -2710,6 +2710,12 @@ func (r *OLAPRepositoryImpl) GetEventWithPayload(ctx context.Context, externalId failedCount = eventData.FailedCount } + var eventScope *string + + if event.Scope.Valid && event.Scope.String != "" { + eventScope = &event.Scope.String + } + return &EventWithPayload{ ListEventsRow: &ListEventsRow{ TenantID: event.TenantID, @@ -2719,7 +2725,7 @@ func (r *OLAPRepositoryImpl) GetEventWithPayload(ctx context.Context, externalId EventKey: event.Key, EventPayload: payload, EventAdditionalMetadata: event.AdditionalMetadata, - EventScope: event.Scope.String, + EventScope: eventScope, QueuedCount: queuedCount, RunningCount: runningCount, CompletedCount: completedCount, @@ -2740,7 +2746,7 @@ type ListEventsRow struct { EventKey string `json:"event_key"` EventPayload []byte `json:"event_payload"` EventAdditionalMetadata []byte `json:"event_additional_metadata"` - EventScope string `json:"event_scope"` + EventScope *string `json:"event_scope,omitempty,omitzero"` QueuedCount int64 `json:"queued_count"` RunningCount int64 `json:"running_count"` CompletedCount int64 `json:"completed_count"` @@ -2793,7 +2799,6 @@ func (r *OLAPRepositoryImpl) ListEvents(ctx context.Context, opts sqlcv1.ListEve eventExternalIds := make([]uuid.UUID, len(events)) readPayloadOpts := make([]ReadOLAPPayloadOpts, len(events)) - minSeenAt := sqlchelpers.TimestamptzFromTime(time.Now()) for i, event := range events { eventExternalIds[i] = event.ExternalID @@ -2801,17 +2806,13 @@ func (r *OLAPRepositoryImpl) ListEvents(ctx context.Context, opts sqlcv1.ListEve ExternalId: event.ExternalID, InsertedAt: event.SeenAt, } - - if event.SeenAt.Time.Before(minSeenAt.Time) { - minSeenAt = event.SeenAt - } } eventExternalIdToData, err := r.PopulateEventData( ctx, opts.Tenantid, eventExternalIds, - minSeenAt, + opts.Since, ) if err != nil { @@ -2854,6 +2855,12 @@ func (r *OLAPRepositoryImpl) ListEvents(ctx context.Context, opts sqlcv1.ListEve failedCount = data.FailedCount } + var eventScope *string + + if event.Scope.Valid && event.Scope.String != "" { + eventScope = &event.Scope.String + } + result = append(result, &EventWithPayload{ ListEventsRow: &ListEventsRow{ TenantID: event.TenantID, @@ -2863,7 +2870,7 @@ func (r *OLAPRepositoryImpl) ListEvents(ctx context.Context, opts sqlcv1.ListEve EventKey: event.Key, EventPayload: payload, EventAdditionalMetadata: event.AdditionalMetadata, - EventScope: event.Scope.String, + EventScope: eventScope, QueuedCount: queuedCount, RunningCount: runningCount, CompletedCount: completedCount, diff --git a/pkg/repository/scheduler.go b/pkg/repository/scheduler.go index 7bb2218ed9..ba5b4cc95a 100644 --- a/pkg/repository/scheduler.go +++ b/pkg/repository/scheduler.go @@ -55,7 +55,7 @@ type OptimisticSchedulingRepository interface { TriggerFromEvents(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []EventTriggerOpts) ([]*sqlcv1.V1QueueItem, *TriggerFromEventsResult, error) - TriggerFromNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*sqlcv1.V1QueueItem, []*V1TaskWithPayload, []*DAGWithData, error) + TriggerFromNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*sqlcv1.V1QueueItem, []*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, error) MarkQueueItemsProcessed(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, r *AssignResults) (succeeded []*AssignedItem, failed []*AssignedItem, err error) } diff --git a/pkg/repository/scheduler_optimistic.go b/pkg/repository/scheduler_optimistic.go index c26796ff3e..a528153fc8 100644 --- a/pkg/repository/scheduler_optimistic.go +++ b/pkg/repository/scheduler_optimistic.go @@ -72,17 +72,17 @@ func (r *optimisticSchedulingRepositoryImpl) TriggerFromEvents(ctx context.Conte return qis, result, nil } -func (r *optimisticSchedulingRepositoryImpl) TriggerFromNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*sqlcv1.V1QueueItem, []*V1TaskWithPayload, []*DAGWithData, error) { +func (r *optimisticSchedulingRepositoryImpl) TriggerFromNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*sqlcv1.V1QueueItem, []*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, error) { triggerOpts, err := r.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) } - tasks, dags, err := r.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) + tasks, dags, idempotencyKeyCollisions, _, err := r.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to trigger workflows: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to trigger workflows: %w", err) } // get the queue items for the tasks that were created @@ -107,11 +107,11 @@ func (r *optimisticSchedulingRepositoryImpl) TriggerFromNames(ctx context.Contex if errors.Is(err, pgx.ErrNoRows) { qis = []*sqlcv1.V1QueueItem{} } else { - return nil, nil, nil, fmt.Errorf("failed to list queue items for tasks: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to list queue items for tasks: %w", err) } } - return qis, tasks, dags, nil + return qis, tasks, dags, idempotencyKeyCollisions, nil } func (r *optimisticSchedulingRepositoryImpl) MarkQueueItemsProcessed(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, r2 *AssignResults) (succeeded []*AssignedItem, failed []*AssignedItem, err error) { diff --git a/pkg/repository/sqlcv1/idempotency-keys.sql b/pkg/repository/sqlcv1/idempotency-keys.sql index d86b710abd..f4196f262d 100644 --- a/pkg/repository/sqlcv1/idempotency-keys.sql +++ b/pkg/repository/sqlcv1/idempotency-keys.sql @@ -1,15 +1,3 @@ --- name: CreateIdempotencyKey :exec -INSERT INTO v1_idempotency_key ( - tenant_id, - key, - expires_at -) -VALUES ( - @tenantId::UUID, - @key::TEXT, - @expiresAt::TIMESTAMPTZ -); - -- name: CleanUpExpiredIdempotencyKeys :exec DELETE FROM v1_idempotency_key WHERE @@ -19,55 +7,57 @@ WHERE -- name: ClaimIdempotencyKeys :many WITH inputs AS ( - SELECT DISTINCT + SELECT UNNEST(@keys::TEXT[]) AS key, + UNNEST(@expiresAts::TIMESTAMPTZ[]) AS expires_at, UNNEST(@claimedByExternalIds::UUID[]) AS claimed_by_external_id -), incoming_claims AS ( - SELECT - *, - ROW_NUMBER() OVER(PARTITION BY key ORDER BY claimed_by_external_id) AS claim_index +), inputs_with_rn AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY key ORDER BY expires_at DESC) AS rn FROM inputs -), candidate_keys AS ( - -- Grab all of the keys that are attempting to be claimed - SELECT - tenant_id, - expires_at, - key, - ROW_NUMBER() OVER(PARTITION BY tenant_id, key ORDER BY expires_at) AS key_index +), deduplicated_potential_claims AS ( + SELECT * + FROM inputs_with_rn + WHERE rn = 1 +), locked_existing_keys AS ( + SELECT * FROM v1_idempotency_key WHERE tenant_id = @tenantId::UUID AND key IN ( SELECT key - FROM incoming_claims + FROM deduplicated_potential_claims ) - AND claimed_by_external_id IS NULL - AND expires_at > NOW() -), to_update AS ( - SELECT - ck.tenant_id, - ck.expires_at, - ck.key, - ic.claimed_by_external_id - FROM candidate_keys ck - JOIN incoming_claims ic ON (ck.key, ck.key_index) = (ic.key, ic.claim_index) - WHERE ck.tenant_id = @tenantId::UUID FOR UPDATE SKIP LOCKED +), claimable_keys AS ( + SELECT * + FROM locked_existing_keys + WHERE expires_at <= NOW() ), claims AS ( - UPDATE v1_idempotency_key k + INSERT INTO v1_idempotency_key (key, expires_at, tenant_id, claimed_by_external_id) + SELECT key, expires_at, @tenantId::UUID, claimed_by_external_id + FROM deduplicated_potential_claims + ON CONFLICT (tenant_id, key) DO UPDATE SET - claimed_by_external_id = u.claimed_by_external_id, - updated_at = NOW() - FROM to_update u - WHERE (u.tenant_id, u.expires_at, u.key) = (k.tenant_id, k.expires_at, k.key) - RETURNING k.* + expires_at = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN EXCLUDED.expires_at + ELSE v1_idempotency_key.expires_at + END, + claimed_by_external_id = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN EXCLUDED.claimed_by_external_id + ELSE v1_idempotency_key.claimed_by_external_id + END, + updated_at = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN NOW() + ELSE v1_idempotency_key.updated_at + END + RETURNING * ) SELECT i.key::TEXT AS key, c.expires_at::TIMESTAMPTZ AS expires_at, - c.claimed_by_external_id IS NOT NULL::BOOLEAN AS was_successfully_claimed, + (c.claimed_by_external_id = i.claimed_by_external_id)::BOOLEAN AS was_successfully_claimed, c.claimed_by_external_id FROM inputs i -LEFT JOIN claims c ON (i.key = c.key AND i.claimed_by_external_id = c.claimed_by_external_id) +LEFT JOIN claims c USING (key) ; diff --git a/pkg/repository/sqlcv1/idempotency-keys.sql.go b/pkg/repository/sqlcv1/idempotency-keys.sql.go index 1ef63348e1..e807e954eb 100644 --- a/pkg/repository/sqlcv1/idempotency-keys.sql.go +++ b/pkg/repository/sqlcv1/idempotency-keys.sql.go @@ -14,63 +14,66 @@ import ( const claimIdempotencyKeys = `-- name: ClaimIdempotencyKeys :many WITH inputs AS ( - SELECT DISTINCT - UNNEST($1::TEXT[]) AS key, - UNNEST($2::UUID[]) AS claimed_by_external_id -), incoming_claims AS ( SELECT - key, claimed_by_external_id, - ROW_NUMBER() OVER(PARTITION BY key ORDER BY claimed_by_external_id) AS claim_index + UNNEST($1::TEXT[]) AS key, + UNNEST($2::TIMESTAMPTZ[]) AS expires_at, + UNNEST($3::UUID[]) AS claimed_by_external_id +), inputs_with_rn AS ( + SELECT key, expires_at, claimed_by_external_id, ROW_NUMBER() OVER (PARTITION BY key ORDER BY expires_at DESC) AS rn FROM inputs -), candidate_keys AS ( - -- Grab all of the keys that are attempting to be claimed - SELECT - tenant_id, - expires_at, - key, - ROW_NUMBER() OVER(PARTITION BY tenant_id, key ORDER BY expires_at) AS key_index +), deduplicated_potential_claims AS ( + SELECT key, expires_at, claimed_by_external_id, rn + FROM inputs_with_rn + WHERE rn = 1 +), locked_existing_keys AS ( + SELECT tenant_id, key, expires_at, claimed_by_external_id, inserted_at, updated_at FROM v1_idempotency_key WHERE - tenant_id = $3::UUID + tenant_id = $4::UUID AND key IN ( SELECT key - FROM incoming_claims + FROM deduplicated_potential_claims ) - AND claimed_by_external_id IS NULL - AND expires_at > NOW() -), to_update AS ( - SELECT - ck.tenant_id, - ck.expires_at, - ck.key, - ic.claimed_by_external_id - FROM candidate_keys ck - JOIN incoming_claims ic ON (ck.key, ck.key_index) = (ic.key, ic.claim_index) - WHERE ck.tenant_id = $3::UUID FOR UPDATE SKIP LOCKED +), claimable_keys AS ( + SELECT tenant_id, key, expires_at, claimed_by_external_id, inserted_at, updated_at + FROM locked_existing_keys + WHERE expires_at <= NOW() ), claims AS ( - UPDATE v1_idempotency_key k + INSERT INTO v1_idempotency_key (key, expires_at, tenant_id, claimed_by_external_id) + SELECT key, expires_at, $4::UUID, claimed_by_external_id + FROM deduplicated_potential_claims + ON CONFLICT (tenant_id, key) DO UPDATE SET - claimed_by_external_id = u.claimed_by_external_id, - updated_at = NOW() - FROM to_update u - WHERE (u.tenant_id, u.expires_at, u.key) = (k.tenant_id, k.expires_at, k.key) - RETURNING k.tenant_id, k.key, k.expires_at, k.claimed_by_external_id, k.inserted_at, k.updated_at + expires_at = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN EXCLUDED.expires_at + ELSE v1_idempotency_key.expires_at + END, + claimed_by_external_id = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN EXCLUDED.claimed_by_external_id + ELSE v1_idempotency_key.claimed_by_external_id + END, + updated_at = CASE + WHEN (v1_idempotency_key.tenant_id, v1_idempotency_key.key) IN (SELECT tenant_id, key FROM claimable_keys) THEN NOW() + ELSE v1_idempotency_key.updated_at + END + RETURNING tenant_id, key, expires_at, claimed_by_external_id, inserted_at, updated_at ) SELECT i.key::TEXT AS key, c.expires_at::TIMESTAMPTZ AS expires_at, - c.claimed_by_external_id IS NOT NULL::BOOLEAN AS was_successfully_claimed, + (c.claimed_by_external_id = i.claimed_by_external_id)::BOOLEAN AS was_successfully_claimed, c.claimed_by_external_id FROM inputs i -LEFT JOIN claims c ON (i.key = c.key AND i.claimed_by_external_id = c.claimed_by_external_id) +LEFT JOIN claims c USING (key) ` type ClaimIdempotencyKeysParams struct { - Keys []string `json:"keys"` - Claimedbyexternalids []uuid.UUID `json:"claimedbyexternalids"` - Tenantid uuid.UUID `json:"tenantid"` + Keys []string `json:"keys"` + Expiresats []pgtype.Timestamptz `json:"expiresats"` + Claimedbyexternalids []uuid.UUID `json:"claimedbyexternalids"` + Tenantid uuid.UUID `json:"tenantid"` } type ClaimIdempotencyKeysRow struct { @@ -81,7 +84,12 @@ type ClaimIdempotencyKeysRow struct { } func (q *Queries) ClaimIdempotencyKeys(ctx context.Context, db DBTX, arg ClaimIdempotencyKeysParams) ([]*ClaimIdempotencyKeysRow, error) { - rows, err := db.Query(ctx, claimIdempotencyKeys, arg.Keys, arg.Claimedbyexternalids, arg.Tenantid) + rows, err := db.Query(ctx, claimIdempotencyKeys, + arg.Keys, + arg.Expiresats, + arg.Claimedbyexternalids, + arg.Tenantid, + ) if err != nil { return nil, err } @@ -116,27 +124,3 @@ func (q *Queries) CleanUpExpiredIdempotencyKeys(ctx context.Context, db DBTX, te _, err := db.Exec(ctx, cleanUpExpiredIdempotencyKeys, tenantid) return err } - -const createIdempotencyKey = `-- name: CreateIdempotencyKey :exec -INSERT INTO v1_idempotency_key ( - tenant_id, - key, - expires_at -) -VALUES ( - $1::UUID, - $2::TEXT, - $3::TIMESTAMPTZ -) -` - -type CreateIdempotencyKeyParams struct { - Tenantid uuid.UUID `json:"tenantid"` - Key string `json:"key"` - Expiresat pgtype.Timestamptz `json:"expiresat"` -} - -func (q *Queries) CreateIdempotencyKey(ctx context.Context, db DBTX, arg CreateIdempotencyKeyParams) error { - _, err := db.Exec(ctx, createIdempotencyKey, arg.Tenantid, arg.Key, arg.Expiresat) - return err -} diff --git a/pkg/repository/sqlcv1/models.go b/pkg/repository/sqlcv1/models.go index 016059f636..c074c3bddc 100644 --- a/pkg/repository/sqlcv1/models.go +++ b/pkg/repository/sqlcv1/models.go @@ -905,8 +905,9 @@ func (ns NullTenantResourceLimitAlertType) Value() (driver.Value, error) { type V1CelEvaluationFailureSource string const ( - V1CelEvaluationFailureSourceFILTER V1CelEvaluationFailureSource = "FILTER" - V1CelEvaluationFailureSourceWEBHOOK V1CelEvaluationFailureSource = "WEBHOOK" + V1CelEvaluationFailureSourceFILTER V1CelEvaluationFailureSource = "FILTER" + V1CelEvaluationFailureSourceWEBHOOK V1CelEvaluationFailureSource = "WEBHOOK" + V1CelEvaluationFailureSourceIDEMPOTENCYKEY V1CelEvaluationFailureSource = "IDEMPOTENCY_KEY" ) func (e *V1CelEvaluationFailureSource) Scan(src interface{}) error { @@ -4012,4 +4013,6 @@ type WorkflowVersion struct { DefaultPriority pgtype.Int4 `json:"defaultPriority"` CreateWorkflowVersionOpts []byte `json:"createWorkflowVersionOpts"` InputJsonSchema []byte `json:"inputJsonSchema"` + IdempotencyKeyExpression pgtype.Text `json:"idempotencyKeyExpression"` + IdempotencyKeyTtlMs pgtype.Int8 `json:"idempotencyKeyTtlMs"` } diff --git a/pkg/repository/sqlcv1/olap.sql b/pkg/repository/sqlcv1/olap.sql index 7b64bcff28..0827efc10c 100644 --- a/pkg/repository/sqlcv1/olap.sql +++ b/pkg/repository/sqlcv1/olap.sql @@ -1875,8 +1875,8 @@ WHERE ) AND ( sqlc.narg('eventIds')::UUID[] IS NULL OR - EXISTS ( - SELECT 1 + (e.id, e.seen_at) IN ( + SELECT event_id, event_seen_at FROM v1_event_lookup_table_olap elt WHERE elt.tenant_id = @tenantId::UUID AND elt.external_id = ANY(sqlc.narg('eventIds')::UUID[]) ) @@ -1937,8 +1937,8 @@ WITH included_events AS ( ) AND ( sqlc.narg('eventIds')::UUID[] IS NULL OR - EXISTS ( - SELECT 1 + (e.id, e.seen_at) IN ( + SELECT event_id, event_seen_at FROM v1_event_lookup_table_olap elt WHERE elt.tenant_id = @tenantId::UUID AND elt.external_id = ANY(sqlc.narg('eventIds')::UUID[]) ) diff --git a/pkg/repository/sqlcv1/olap.sql.go b/pkg/repository/sqlcv1/olap.sql.go index d772d28d90..3ac64aba86 100644 --- a/pkg/repository/sqlcv1/olap.sql.go +++ b/pkg/repository/sqlcv1/olap.sql.go @@ -243,8 +243,8 @@ WITH included_events AS ( ) AND ( $6::UUID[] IS NULL OR - EXISTS ( - SELECT 1 + (e.id, e.seen_at) IN ( + SELECT event_id, event_seen_at FROM v1_event_lookup_table_olap elt WHERE elt.tenant_id = $1::UUID AND elt.external_id = ANY($6::UUID[]) ) @@ -1334,8 +1334,8 @@ WHERE ) AND ( $6::UUID[] IS NULL OR - EXISTS ( - SELECT 1 + (e.id, e.seen_at) IN ( + SELECT event_id, event_seen_at FROM v1_event_lookup_table_olap elt WHERE elt.tenant_id = $1::UUID AND elt.external_id = ANY($6::UUID[]) ) diff --git a/pkg/repository/sqlcv1/tasks.sql b/pkg/repository/sqlcv1/tasks.sql index 163292ee09..38e38fc796 100644 --- a/pkg/repository/sqlcv1/tasks.sql +++ b/pkg/repository/sqlcv1/tasks.sql @@ -1377,7 +1377,9 @@ SELECT run_external_id, event_id, event_seen_at, - filter_id + CASE WHEN filter_id = '00000000-0000-0000-0000-000000000000'::uuid THEN NULL + ELSE filter_id + END AS filter_id FROM input RETURNING diff --git a/pkg/repository/sqlcv1/tasks.sql.go b/pkg/repository/sqlcv1/tasks.sql.go index 0bff210b4f..d555382cfd 100644 --- a/pkg/repository/sqlcv1/tasks.sql.go +++ b/pkg/repository/sqlcv1/tasks.sql.go @@ -187,7 +187,9 @@ SELECT run_external_id, event_id, event_seen_at, - filter_id + CASE WHEN filter_id = '00000000-0000-0000-0000-000000000000'::uuid THEN NULL + ELSE filter_id + END AS filter_id FROM input RETURNING diff --git a/pkg/repository/sqlcv1/triggers.sql b/pkg/repository/sqlcv1/triggers.sql index f1e9b90508..aabbe79639 100644 --- a/pkg/repository/sqlcv1/triggers.sql +++ b/pkg/repository/sqlcv1/triggers.sql @@ -4,7 +4,9 @@ WITH latest_versions AS ( SELECT DISTINCT ON("workflowId") "workflowId", workflowVersions."id" AS "workflowVersionId", - workflow."name" AS "workflowName" + workflow."name" AS "workflowName", + workflowVersions."idempotencyKeyExpression", + workflowVersions."idempotencyKeyTtlMs" FROM "WorkflowVersion" as workflowVersions JOIN @@ -24,7 +26,9 @@ SELECT latest_versions."workflowId", latest_versions."workflowName", eventRef."eventKey" as "workflowTriggeringEventKeyPattern", - k.event_key::TEXT as "incomingEventKey" + k.event_key::TEXT as "incomingEventKey", + latest_versions."idempotencyKeyExpression", + latest_versions."idempotencyKeyTtlMs" FROM latest_versions JOIN @@ -38,7 +42,9 @@ JOIN event_keys k ON k.event_key LIKE REPLACE(eventRef."eventKey", '*', '%') SELECT DISTINCT ON("workflowId") "workflowId", workflowVersions."id" AS "workflowVersionId", - workflow."name" AS "workflowName" + workflow."name" AS "workflowName", + workflowVersions."idempotencyKeyExpression", + workflowVersions."idempotencyKeyTtlMs" FROM "WorkflowVersion" as workflowVersions JOIN diff --git a/pkg/repository/sqlcv1/triggers.sql.go b/pkg/repository/sqlcv1/triggers.sql.go index 1b6d856f65..f15c6829d8 100644 --- a/pkg/repository/sqlcv1/triggers.sql.go +++ b/pkg/repository/sqlcv1/triggers.sql.go @@ -9,13 +9,16 @@ import ( "context" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" ) const listWorkflowsByNames = `-- name: ListWorkflowsByNames :many SELECT DISTINCT ON("workflowId") "workflowId", workflowVersions."id" AS "workflowVersionId", - workflow."name" AS "workflowName" + workflow."name" AS "workflowName", + workflowVersions."idempotencyKeyExpression", + workflowVersions."idempotencyKeyTtlMs" FROM "WorkflowVersion" as workflowVersions JOIN @@ -33,9 +36,11 @@ type ListWorkflowsByNamesParams struct { } type ListWorkflowsByNamesRow struct { - WorkflowId uuid.UUID `json:"workflowId"` - WorkflowVersionId uuid.UUID `json:"workflowVersionId"` - WorkflowName string `json:"workflowName"` + WorkflowId uuid.UUID `json:"workflowId"` + WorkflowVersionId uuid.UUID `json:"workflowVersionId"` + WorkflowName string `json:"workflowName"` + IdempotencyKeyExpression pgtype.Text `json:"idempotencyKeyExpression"` + IdempotencyKeyTtlMs pgtype.Int8 `json:"idempotencyKeyTtlMs"` } func (q *Queries) ListWorkflowsByNames(ctx context.Context, db DBTX, arg ListWorkflowsByNamesParams) ([]*ListWorkflowsByNamesRow, error) { @@ -47,7 +52,13 @@ func (q *Queries) ListWorkflowsByNames(ctx context.Context, db DBTX, arg ListWor var items []*ListWorkflowsByNamesRow for rows.Next() { var i ListWorkflowsByNamesRow - if err := rows.Scan(&i.WorkflowId, &i.WorkflowVersionId, &i.WorkflowName); err != nil { + if err := rows.Scan( + &i.WorkflowId, + &i.WorkflowVersionId, + &i.WorkflowName, + &i.IdempotencyKeyExpression, + &i.IdempotencyKeyTtlMs, + ); err != nil { return nil, err } items = append(items, &i) @@ -63,7 +74,9 @@ WITH latest_versions AS ( SELECT DISTINCT ON("workflowId") "workflowId", workflowVersions."id" AS "workflowVersionId", - workflow."name" AS "workflowName" + workflow."name" AS "workflowName", + workflowVersions."idempotencyKeyExpression", + workflowVersions."idempotencyKeyTtlMs" FROM "WorkflowVersion" as workflowVersions JOIN @@ -82,7 +95,9 @@ SELECT latest_versions."workflowId", latest_versions."workflowName", eventRef."eventKey" as "workflowTriggeringEventKeyPattern", - k.event_key::TEXT as "incomingEventKey" + k.event_key::TEXT as "incomingEventKey", + latest_versions."idempotencyKeyExpression", + latest_versions."idempotencyKeyTtlMs" FROM latest_versions JOIN @@ -98,11 +113,13 @@ type ListWorkflowsForEventsParams struct { } type ListWorkflowsForEventsRow struct { - WorkflowVersionId uuid.UUID `json:"workflowVersionId"` - WorkflowId uuid.UUID `json:"workflowId"` - WorkflowName string `json:"workflowName"` - WorkflowTriggeringEventKeyPattern string `json:"workflowTriggeringEventKeyPattern"` - IncomingEventKey string `json:"incomingEventKey"` + WorkflowVersionId uuid.UUID `json:"workflowVersionId"` + WorkflowId uuid.UUID `json:"workflowId"` + WorkflowName string `json:"workflowName"` + WorkflowTriggeringEventKeyPattern string `json:"workflowTriggeringEventKeyPattern"` + IncomingEventKey string `json:"incomingEventKey"` + IdempotencyKeyExpression pgtype.Text `json:"idempotencyKeyExpression"` + IdempotencyKeyTtlMs pgtype.Int8 `json:"idempotencyKeyTtlMs"` } // Get all of the latest workflow versions @@ -122,6 +139,8 @@ func (q *Queries) ListWorkflowsForEvents(ctx context.Context, db DBTX, arg ListW &i.WorkflowName, &i.WorkflowTriggeringEventKeyPattern, &i.IncomingEventKey, + &i.IdempotencyKeyExpression, + &i.IdempotencyKeyTtlMs, ); err != nil { return nil, err } diff --git a/pkg/repository/sqlcv1/workflows.sql b/pkg/repository/sqlcv1/workflows.sql index a0bbd6cd44..b24ce3d073 100644 --- a/pkg/repository/sqlcv1/workflows.sql +++ b/pkg/repository/sqlcv1/workflows.sql @@ -121,7 +121,9 @@ INSERT INTO "WorkflowVersion" ( "kind", "defaultPriority", "createWorkflowVersionOpts", - "inputJsonSchema" + "inputJsonSchema", + "idempotencyKeyExpression", + "idempotencyKeyTtlMs" ) VALUES ( @id::uuid, coalesce(sqlc.narg('createdAt')::timestamp, CURRENT_TIMESTAMP), @@ -136,7 +138,9 @@ INSERT INTO "WorkflowVersion" ( coalesce(sqlc.narg('kind')::"WorkflowKind", 'DAG'), sqlc.narg('defaultPriority') :: integer, sqlc.narg('createWorkflowVersionOpts')::jsonb, - sqlc.narg('inputJsonSchema')::jsonb + sqlc.narg('inputJsonSchema')::jsonb, + sqlc.narg('idempotencyKeyExpression')::text, + sqlc.narg('idempotencyKeyTtlMs')::bigint ) RETURNING *; -- name: CreateJob :one diff --git a/pkg/repository/sqlcv1/workflows.sql.go b/pkg/repository/sqlcv1/workflows.sql.go index eeec44eaaa..ac650e6f49 100644 --- a/pkg/repository/sqlcv1/workflows.sql.go +++ b/pkg/repository/sqlcv1/workflows.sql.go @@ -915,7 +915,9 @@ INSERT INTO "WorkflowVersion" ( "kind", "defaultPriority", "createWorkflowVersionOpts", - "inputJsonSchema" + "inputJsonSchema", + "idempotencyKeyExpression", + "idempotencyKeyTtlMs" ) VALUES ( $1::uuid, coalesce($2::timestamp, CURRENT_TIMESTAMP), @@ -930,8 +932,10 @@ INSERT INTO "WorkflowVersion" ( coalesce($9::"WorkflowKind", 'DAG'), $10 :: integer, $11::jsonb, - $12::jsonb -) RETURNING id, "createdAt", "updatedAt", "deletedAt", version, "order", "workflowId", checksum, "scheduleTimeout", "onFailureJobId", sticky, kind, "defaultPriority", "createWorkflowVersionOpts", "inputJsonSchema" + $12::jsonb, + $13::text, + $14::bigint +) RETURNING id, "createdAt", "updatedAt", "deletedAt", version, "order", "workflowId", checksum, "scheduleTimeout", "onFailureJobId", sticky, kind, "defaultPriority", "createWorkflowVersionOpts", "inputJsonSchema", "idempotencyKeyExpression", "idempotencyKeyTtlMs" ` type CreateWorkflowVersionParams struct { @@ -947,6 +951,8 @@ type CreateWorkflowVersionParams struct { DefaultPriority pgtype.Int4 `json:"defaultPriority"` CreateWorkflowVersionOpts []byte `json:"createWorkflowVersionOpts"` InputJsonSchema []byte `json:"inputJsonSchema"` + IdempotencyKeyExpression pgtype.Text `json:"idempotencyKeyExpression"` + IdempotencyKeyTtlMs pgtype.Int8 `json:"idempotencyKeyTtlMs"` } func (q *Queries) CreateWorkflowVersion(ctx context.Context, db DBTX, arg CreateWorkflowVersionParams) (*WorkflowVersion, error) { @@ -963,6 +969,8 @@ func (q *Queries) CreateWorkflowVersion(ctx context.Context, db DBTX, arg Create arg.DefaultPriority, arg.CreateWorkflowVersionOpts, arg.InputJsonSchema, + arg.IdempotencyKeyExpression, + arg.IdempotencyKeyTtlMs, ) var i WorkflowVersion err := row.Scan( @@ -981,6 +989,8 @@ func (q *Queries) CreateWorkflowVersion(ctx context.Context, db DBTX, arg Create &i.DefaultPriority, &i.CreateWorkflowVersionOpts, &i.InputJsonSchema, + &i.IdempotencyKeyExpression, + &i.IdempotencyKeyTtlMs, ) return &i, err } @@ -1234,7 +1244,7 @@ func (q *Queries) GetWorkflowShape(ctx context.Context, db DBTX, workflowversion const getWorkflowVersionById = `-- name: GetWorkflowVersionById :one SELECT - wv.id, wv."createdAt", wv."updatedAt", wv."deletedAt", wv.version, wv."order", wv."workflowId", wv.checksum, wv."scheduleTimeout", wv."onFailureJobId", wv.sticky, wv.kind, wv."defaultPriority", wv."createWorkflowVersionOpts", wv."inputJsonSchema", + wv.id, wv."createdAt", wv."updatedAt", wv."deletedAt", wv.version, wv."order", wv."workflowId", wv.checksum, wv."scheduleTimeout", wv."onFailureJobId", wv.sticky, wv.kind, wv."defaultPriority", wv."createWorkflowVersionOpts", wv."inputJsonSchema", wv."idempotencyKeyExpression", wv."idempotencyKeyTtlMs", w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, w."isPaused" FROM "WorkflowVersion" as wv @@ -1269,6 +1279,8 @@ func (q *Queries) GetWorkflowVersionById(ctx context.Context, db DBTX, id uuid.U &i.WorkflowVersion.DefaultPriority, &i.WorkflowVersion.CreateWorkflowVersionOpts, &i.WorkflowVersion.InputJsonSchema, + &i.WorkflowVersion.IdempotencyKeyExpression, + &i.WorkflowVersion.IdempotencyKeyTtlMs, &i.Workflow.ID, &i.Workflow.CreatedAt, &i.Workflow.UpdatedAt, @@ -1357,7 +1369,7 @@ func (q *Queries) GetWorkflowVersionEventTriggerRefs(ctx context.Context, db DBT const getWorkflowVersionForEngine = `-- name: GetWorkflowVersionForEngine :many SELECT - workflowversions.id, workflowversions."createdAt", workflowversions."updatedAt", workflowversions."deletedAt", workflowversions.version, workflowversions."order", workflowversions."workflowId", workflowversions.checksum, workflowversions."scheduleTimeout", workflowversions."onFailureJobId", workflowversions.sticky, workflowversions.kind, workflowversions."defaultPriority", workflowversions."createWorkflowVersionOpts", workflowversions."inputJsonSchema", + workflowversions.id, workflowversions."createdAt", workflowversions."updatedAt", workflowversions."deletedAt", workflowversions.version, workflowversions."order", workflowversions."workflowId", workflowversions.checksum, workflowversions."scheduleTimeout", workflowversions."onFailureJobId", workflowversions.sticky, workflowversions.kind, workflowversions."defaultPriority", workflowversions."createWorkflowVersionOpts", workflowversions."inputJsonSchema", workflowversions."idempotencyKeyExpression", workflowversions."idempotencyKeyTtlMs", w."name" as "workflowName", wc."limitStrategy" as "concurrencyLimitStrategy", wc."maxRuns" as "concurrencyMaxRuns", @@ -1415,6 +1427,8 @@ func (q *Queries) GetWorkflowVersionForEngine(ctx context.Context, db DBTX, arg &i.WorkflowVersion.DefaultPriority, &i.WorkflowVersion.CreateWorkflowVersionOpts, &i.WorkflowVersion.InputJsonSchema, + &i.WorkflowVersion.IdempotencyKeyExpression, + &i.WorkflowVersion.IdempotencyKeyTtlMs, &i.WorkflowName, &i.ConcurrencyLimitStrategy, &i.ConcurrencyMaxRuns, @@ -1481,7 +1495,7 @@ const linkOnFailureJob = `-- name: LinkOnFailureJob :one UPDATE "WorkflowVersion" SET "onFailureJobId" = $1::uuid WHERE "id" = $2::uuid -RETURNING id, "createdAt", "updatedAt", "deletedAt", version, "order", "workflowId", checksum, "scheduleTimeout", "onFailureJobId", sticky, kind, "defaultPriority", "createWorkflowVersionOpts", "inputJsonSchema" +RETURNING id, "createdAt", "updatedAt", "deletedAt", version, "order", "workflowId", checksum, "scheduleTimeout", "onFailureJobId", sticky, kind, "defaultPriority", "createWorkflowVersionOpts", "inputJsonSchema", "idempotencyKeyExpression", "idempotencyKeyTtlMs" ` type LinkOnFailureJobParams struct { @@ -1508,6 +1522,8 @@ func (q *Queries) LinkOnFailureJob(ctx context.Context, db DBTX, arg LinkOnFailu &i.DefaultPriority, &i.CreateWorkflowVersionOpts, &i.InputJsonSchema, + &i.IdempotencyKeyExpression, + &i.IdempotencyKeyTtlMs, ) return &i, err } diff --git a/pkg/repository/trigger.go b/pkg/repository/trigger.go index 5f33141438..f9aca9ee72 100644 --- a/pkg/repository/trigger.go +++ b/pkg/repository/trigger.go @@ -139,10 +139,12 @@ type createDAGOpts struct { type TriggerRepository interface { TriggerFromEvents(ctx context.Context, tenantId uuid.UUID, opts []EventTriggerOpts) (*TriggerFromEventsResult, error) - TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, error) + TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, []CELEvaluationFailure, error) PopulateExternalIdsForWorkflow(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) error + PopulateWorkflowIdempotencyPresence(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) error + PreflightVerifyWorkflowNameOpts(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) error NewTriggerTaskData(ctx context.Context, tenantId uuid.UUID, req *v1contracts.TriggerWorkflowRequest, parentTask *sqlcv1.FlattenExternalIdsRow) (*TriggerTaskData, error) @@ -312,7 +314,7 @@ func (r *sharedRepository) doTriggerFromEvents( return nil, fmt.Errorf("failed to prepare trigger from events: %w", err) } - tasks, dags, err := r.triggerWorkflows(ctx, tx, tenantId, triggerOpts, createCoreEventOpts) + tasks, dags, _, workflowCelFailures, err := r.triggerWorkflows(ctx, tx, tenantId, triggerOpts, createCoreEventOpts) if err != nil { return nil, fmt.Errorf("failed to trigger workflows: %w", err) @@ -324,7 +326,7 @@ func (r *sharedRepository) doTriggerFromEvents( Tasks: tasks, Dags: dags, EventExternalIdToRuns: eventExternalIdToRuns, - CELEvaluationFailures: celEvaluationFailures, + CELEvaluationFailures: append(celEvaluationFailures, workflowCelFailures...), }, nil } @@ -372,36 +374,36 @@ func getEventExternalIdToRuns(opts []EventTriggerOpts, externalIdToEventIdAndFil return eventExternalIdToRuns } -func (s *sharedRepository) triggerFromWorkflowNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, error) { +func (s *sharedRepository) triggerFromWorkflowNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, []CELEvaluationFailure, error) { triggerOpts, err := s.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) if err != nil { - return nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) } return s.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) } -func (r *TriggerRepositoryImpl) TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, error) { +func (r *TriggerRepositoryImpl) TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, []CELEvaluationFailure, error) { tx, err := r.PrepareOptimisticTx(ctx) if err != nil { - return nil, nil, fmt.Errorf("failed to prepare tx: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to prepare tx: %w", err) } defer tx.Rollback() - tasks, dags, err := r.triggerFromWorkflowNames(ctx, tx, tenantId, opts) + tasks, dags, idempotencyKeyCollisions, celEvaluationFailures, err := r.triggerFromWorkflowNames(ctx, tx, tenantId, opts) if err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } if err := tx.Commit(ctx); err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } - return tasks, dags, nil + return tasks, dags, idempotencyKeyCollisions, celEvaluationFailures, nil } type ErrNamesNotFound struct { @@ -453,6 +455,48 @@ func (r *TriggerRepositoryImpl) PreflightVerifyWorkflowNameOpts(ctx context.Cont return nil } +func (r *TriggerRepositoryImpl) PopulateWorkflowIdempotencyPresence(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) error { + if len(opts) == 0 { + return nil + } + + uniqueNames := make(map[string]struct{}) + names := make([]string, 0, len(uniqueNames)) + + for _, opt := range opts { + if _, ok := uniqueNames[opt.WorkflowName]; !ok { + names = append(names, opt.WorkflowName) + } + + uniqueNames[opt.WorkflowName] = struct{}{} + } + + rows, err := r.listWorkflowsByNames(ctx, r.pool, tenantId, names) + + if err != nil { + return fmt.Errorf("failed to list workflows by names: %w", err) + } + + nameToRow := make(map[string]*sqlcv1.ListWorkflowsByNamesRow, len(rows)) + + for _, row := range rows { + nameToRow[row.WorkflowName] = row + } + + for _, opt := range opts { + row, ok := nameToRow[opt.WorkflowName] + + if !ok { + r.l.Error().Str("workflowName", opt.WorkflowName).Msg("workflow name not found when populating idempotency presence") + continue + } + + opt.HasIdempotencyKey = row.IdempotencyKeyExpression.Valid && row.IdempotencyKeyTtlMs.Valid + } + + return nil +} + type TriggeredBy interface { ToMetadata([]byte) []byte } @@ -588,6 +632,7 @@ type triggerTuple struct { desiredWorkerLabels []*sqlcv1.GetDesiredLabelsRow triggeringEventExternalId *uuid.UUID triggeringEventKey *string + idempotency *IdempotencyConfig } type createCoreUserEventOpts struct { @@ -596,21 +641,135 @@ type createCoreUserEventOpts struct { params sqlcv1.BulkCreateEventsParams } +const internalIdempotencyKeyPrefix = "hatchet_internal_" + +func (r *sharedRepository) evalIdempotencyKey(tuple triggerTuple) (string, error) { + inputData := make(map[string]any) + if len(tuple.input) > 0 { + if err := json.Unmarshal(tuple.input, &inputData); err != nil { + return "", fmt.Errorf("failed to unmarshal input for idempotency key evaluation: %w", err) + } + } + + additionalMetadata := make(map[string]any) + if len(tuple.additionalMetadata) > 0 { + if err := json.Unmarshal(tuple.additionalMetadata, &additionalMetadata); err != nil { + return "", fmt.Errorf("failed to unmarshal additional metadata for idempotency key evaluation: %w", err) + } + } + + key, err := r.celParser.ParseAndEvalWorkflowString( + tuple.idempotency.Expression, + cel.NewInput( + cel.WithInput(inputData), + cel.WithAdditionalMetadata(additionalMetadata), + ), + ) + if err != nil { + return "", fmt.Errorf("failed to evaluate idempotency key expression %q: %w", tuple.idempotency.Expression, err) + } + + if strings.HasPrefix(key, internalIdempotencyKeyPrefix) { + return "", fmt.Errorf("idempotency key %q uses reserved prefix %q", key, internalIdempotencyKeyPrefix) + } + + return key, nil +} + func (r *sharedRepository) triggerWorkflows( ctx context.Context, existingTx *OptimisticTx, tenantId uuid.UUID, - tuples []triggerTuple, + triggerCandidateTuples []triggerTuple, coreEvents *createCoreUserEventOpts, -) ([]*V1TaskWithPayload, []*DAGWithData, error) { - for i := range tuples { - tuples[i].additionalMetadata = ensureTraceparent(tuples[i].additionalMetadata, tuples[i].externalId) +) ([]*V1TaskWithPayload, []*DAGWithData, []IdempotencyCollision, []CELEvaluationFailure, error) { + var preflightTx sqlcv1.DBTX = r.pool + + if existingTx != nil { + preflightTx = existingTx.tx + } + + tuples := make([]triggerTuple, 0, len(triggerCandidateTuples)) + + keys := make([]string, 0, len(triggerCandidateTuples)) + expiresAts := make([]pgtype.Timestamptz, 0, len(triggerCandidateTuples)) + claimedByExternalIds := make([]uuid.UUID, 0, len(triggerCandidateTuples)) + externalIdToTuple := make(map[uuid.UUID]triggerTuple) + + var celEvaluationFailures []CELEvaluationFailure + + for _, tuple := range triggerCandidateTuples { + if tuple.idempotency == nil { + tuples = append(tuples, tuple) + } else { + externalIdToTuple[tuple.externalId] = tuple + key, err := r.evalIdempotencyKey(tuple) + if err != nil { + r.l.Error().Ctx(ctx).Err(err).Msg("failed to evaluate idempotency key, skipping tuple") + celEvaluationFailures = append(celEvaluationFailures, CELEvaluationFailure{ + Source: sqlcv1.V1CelEvaluationFailureSourceIDEMPOTENCYKEY, + ErrorMessage: err.Error(), + }) + continue + } + + keys = append(keys, key) + expiresAts = append(expiresAts, pgtype.Timestamptz{ + Time: time.Now().Add(time.Duration(tuple.idempotency.TTLMs) * time.Millisecond), + Valid: true, + }) + claimedByExternalIds = append(claimedByExternalIds, tuple.externalId) + } + } + + var idempotencyKeyCollisions []IdempotencyCollision + + if len(keys) > 0 { + claims, err := r.queries.ClaimIdempotencyKeys(ctx, preflightTx, sqlcv1.ClaimIdempotencyKeysParams{ + Keys: keys, + Expiresats: expiresAts, + Claimedbyexternalids: claimedByExternalIds, + Tenantid: tenantId, + }) + + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed to claim idempotency keys: %w", err) + } + + idempotencyKeyToLockHolder := make(map[string]uuid.UUID, len(claims)) + for _, claim := range claims { + if claim.ClaimedByExternalID != nil { + idempotencyKeyToLockHolder[claim.Key] = *claim.ClaimedByExternalID + } + } + + for i, key := range keys { + requestedId := claimedByExternalIds[i] + lockHolderExternalId, ok := idempotencyKeyToLockHolder[key] + if !ok { + continue + } + + if lockHolderExternalId == requestedId { + tuple, ok := externalIdToTuple[requestedId] + if !ok { + continue + } + tuples = append(tuples, tuple) + } else { + idempotencyKeyCollisions = append(idempotencyKeyCollisions, IdempotencyCollision{ + RequestedExternalId: requestedId, + ExistingExternalId: lockHolderExternalId, + }) + } + } } // get unique workflow version ids uniqueWorkflowVersionIds := make(map[uuid.UUID]struct{}) - for _, tuple := range tuples { + for i, tuple := range tuples { + tuples[i].additionalMetadata = ensureTraceparent(tuples[i].additionalMetadata, tuples[i].externalId) uniqueWorkflowVersionIds[tuple.workflowVersionId] = struct{}{} } @@ -621,16 +780,10 @@ func (r *sharedRepository) triggerWorkflows( workflowVersionIds = append(workflowVersionIds, id) } - var preflightTx sqlcv1.DBTX = r.pool - - if existingTx != nil { - preflightTx = existingTx.tx - } - workflowVersionToSteps, err := r.listStepsByWorkflowVersionIds(ctx, preflightTx, tenantId, workflowVersionIds) if err != nil { - return nil, nil, fmt.Errorf("failed to get workflow versions for engine: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to get workflow versions for engine: %w", err) } // group steps by workflow version ids @@ -665,7 +818,7 @@ func (r *sharedRepository) triggerWorkflows( preTask, postTask := r.m.Meter(ctx, preflightTx, sqlcv1.LimitResourceTASKRUN, tenantId, int32(countTasks)) // nolint: gosec if err := preTask(); err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } stepsToAdditionalMatches := make(map[uuid.UUID][]*sqlcv1.V1StepMatchCondition) @@ -677,7 +830,7 @@ func (r *sharedRepository) triggerWorkflows( }) if err != nil { - return nil, nil, fmt.Errorf("failed to list step match conditions: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to list step match conditions: %w", err) } for _, match := range additionalMatches { @@ -758,7 +911,7 @@ func (r *sharedRepository) triggerWorkflows( tx, commit, rollback, err = sqlchelpers.PrepareTx(ctx, r.pool, r.l) if err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } defer rollback() @@ -771,7 +924,7 @@ func (r *sharedRepository) triggerWorkflows( tuplesToSkip, err := r.registerChildWorkflows(ctx, tx, tenantId, tuples, stepsToExternalIds, workflowVersionToSteps) if err != nil { - return nil, nil, fmt.Errorf("failed to register child workflows: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to register child workflows: %w", err) } for i, tuple := range tuples { @@ -917,7 +1070,7 @@ func (r *sharedRepository) triggerWorkflows( ) if err != nil { - return nil, nil, fmt.Errorf("failed to create sleep condition: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create sleep condition: %w", err) } groupConditions = append(groupConditions, *c) @@ -1174,7 +1327,7 @@ func (r *sharedRepository) triggerWorkflows( dags, err := r.createDAGs(ctx, tx, tenantId, dagOpts) if err != nil { - return nil, nil, fmt.Errorf("failed to create DAGs: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create DAGs: %w", err) } // populate taskOpts with inserted DAG data @@ -1199,7 +1352,7 @@ func (r *sharedRepository) triggerWorkflows( tasks, err := r.createTasks(ctx, tx, tenantId, createTaskOpts) if err != nil { - return nil, nil, fmt.Errorf("failed to create tasks: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create tasks: %w", err) } for _, dag := range dags { @@ -1216,7 +1369,7 @@ func (r *sharedRepository) triggerWorkflows( err = r.createEventMatches(ctx, tx, tenantId, createMatchOpts) if err != nil { - return nil, nil, fmt.Errorf("failed to create event matches: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create event matches: %w", err) } storePayloadOpts := make([]StorePayloadOpts, 0, len(tasks)) @@ -1249,7 +1402,7 @@ func (r *sharedRepository) triggerWorkflows( createdEvents, err := r.queries.BulkCreateEvents(ctx, tx, coreEvents.params) if err != nil { - return nil, nil, fmt.Errorf("failed to create core events: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create core events: %w", err) } for _, createdEvent := range createdEvents { @@ -1325,7 +1478,7 @@ func (r *sharedRepository) triggerWorkflows( }) if err != nil { - return nil, nil, fmt.Errorf("failed to create event to runs: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create event to runs: %w", err) } for _, e := range createdEvents { @@ -1349,13 +1502,13 @@ func (r *sharedRepository) triggerWorkflows( err = r.payloadStore.Store(ctx, tx, storePayloadOpts...) if err != nil { - return nil, nil, fmt.Errorf("failed to store payloads: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to store payloads: %w", err) } // commit if we started the transaction if existingTx == nil { if err := commit(ctx); err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } postTask() @@ -1364,7 +1517,7 @@ func (r *sharedRepository) triggerWorkflows( existingTx.AddPostCommit(postTask) } - return tasks, dags, nil + return tasks, dags, idempotencyKeyCollisions, celEvaluationFailures, nil } type DAGWithData struct { @@ -2238,6 +2391,15 @@ func (r *sharedRepository) prepareTriggerFromEvents(ctx context.Context, tx sqlc additionalMetadata := triggerConverter.ToMetadata(opt.AdditionalMetadata) externalId := uuid.New() + var idempotency *IdempotencyConfig + + if workflow.IdempotencyKeyExpression.Valid && workflow.IdempotencyKeyTtlMs.Valid { + idempotency = &IdempotencyConfig{ + Expression: workflow.IdempotencyKeyExpression.String, + TTLMs: workflow.IdempotencyKeyTtlMs.Int64, + } + } + triggerOpts = append(triggerOpts, triggerTuple{ workflowVersionId: workflow.WorkflowVersionId, workflowId: workflow.WorkflowId, @@ -2249,6 +2411,7 @@ func (r *sharedRepository) prepareTriggerFromEvents(ctx context.Context, tx sqlc filterPayload: decision.FilterPayload, triggeringEventExternalId: &opt.ExternalId, triggeringEventKey: &opt.Key, + idempotency: idempotency, }) externalIdToEventIdAndFilterId[externalId] = EventExternalIdFilterId{ @@ -2283,13 +2446,8 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, workflowNames := make([]string, 0, len(opts)) uniqueNames := make(map[string]struct{}) namesToOpts := make(map[string][]*WorkflowNameTriggerOpts) - idempotencyKeyToExternalIds := make(map[IdempotencyKey]uuid.UUID) for _, opt := range opts { - if opt.IdempotencyKey != nil { - idempotencyKeyToExternalIds[*opt.IdempotencyKey] = opt.ExternalId - } - namesToOpts[opt.WorkflowName] = append(namesToOpts[opt.WorkflowName], opt) if _, ok := uniqueNames[opt.WorkflowName]; ok { @@ -2300,21 +2458,6 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, workflowNames = append(workflowNames, opt.WorkflowName) } - keyClaimantPairs := make([]KeyClaimantPair, 0, len(idempotencyKeyToExternalIds)) - - for idempotencyKey, runExternalId := range idempotencyKeyToExternalIds { - keyClaimantPairs = append(keyClaimantPairs, KeyClaimantPair{ - IdempotencyKey: idempotencyKey, - ClaimedByExternalId: runExternalId, - }) - } - - keyClaimantPairToWasClaimed, err := claimIdempotencyKeys(ctx, r.queries, tx, tenantId, keyClaimantPairs) - - if err != nil { - return nil, fmt.Errorf("failed to claim idempotency keys: %w", err) - } - workflowVersionsByNames, err := r.listWorkflowsByNames(ctx, tx, tenantId, workflowNames) if err != nil { @@ -2332,17 +2475,11 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, } for _, opt := range opts { - if opt.IdempotencyKey != nil { - keyClaimantPair := KeyClaimantPair{ - IdempotencyKey: *opt.IdempotencyKey, - ClaimedByExternalId: opt.ExternalId, - } - - wasSuccessfullyClaimed := keyClaimantPairToWasClaimed[keyClaimantPair] - - // if we did not successfully claim the idempotency key, we should not trigger the workflow - if !wasSuccessfullyClaimed { - continue + var idempotency *IdempotencyConfig + if workflowVersion.IdempotencyKeyExpression.Valid && workflowVersion.IdempotencyKeyTtlMs.Valid { + idempotency = &IdempotencyConfig{ + Expression: workflowVersion.IdempotencyKeyExpression.String, + TTLMs: workflowVersion.IdempotencyKeyTtlMs.Int64, } } @@ -2361,6 +2498,7 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, childKey: opt.ChildKey, priority: opt.Priority, desiredWorkerLabels: opt.DesiredWorkerLabels, + idempotency: idempotency, }) } } diff --git a/pkg/repository/workflow.go b/pkg/repository/workflow.go index 0525eeaaeb..e0bbda6e4f 100644 --- a/pkg/repository/workflow.go +++ b/pkg/repository/workflow.go @@ -56,6 +56,13 @@ type CreateWorkflowVersionOpts struct { DefaultFilters []types.DefaultFilter `json:"defaultFilters,omitempty" validate:"omitempty,dive"` InputJsonSchema []byte `json:"inputJsonSchema,omitempty"` + + Idempotency *IdempotencyConfig `json:"idempotency,omitempty"` +} + +type IdempotencyConfig struct { + Expression string `json:"expression" validate:"required,celworkflowrunstr"` + TTLMs int64 `json:"ttlMs" validate:"required,min=1"` } type CreateConcurrencyOpts struct { @@ -427,6 +434,15 @@ func (r *workflowRepository) createWorkflowVersionTxs(ctx context.Context, tx sq } } + if opts.Idempotency != nil { + idempotency := *opts.Idempotency + createParams.IdempotencyKeyExpression = pgtype.Text{ + String: idempotency.Expression, + Valid: true, + } + createParams.IdempotencyKeyTtlMs = sqlchelpers.ToBigInt(&idempotency.TTLMs) + } + if opts.DefaultPriority != nil { createParams.DefaultPriority = pgtype.Int4{ Int32: *opts.DefaultPriority, diff --git a/pkg/scheduling/v1/pool.go b/pkg/scheduling/v1/pool.go index 1d4fa94e99..c2dc1a88e8 100644 --- a/pkg/scheduling/v1/pool.go +++ b/pkg/scheduling/v1/pool.go @@ -250,9 +250,9 @@ func (p *SchedulingPool) getTenantManager(tenantId uuid.UUID, storeIfNotFound bo var ErrTenantNotFound = fmt.Errorf("tenant not found in pool") var ErrNoOptimisticSlots = fmt.Errorf("no optimistic slots for scheduling") -func (p *SchedulingPool) RunOptimisticScheduling(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts, localWorkerIds map[uuid.UUID]struct{}) (map[uuid.UUID][]*AssignedItemWithTask, []*v1.V1TaskWithPayload, []*v1.DAGWithData, error) { +func (p *SchedulingPool) RunOptimisticScheduling(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts, localWorkerIds map[uuid.UUID]struct{}) (map[uuid.UUID][]*AssignedItemWithTask, []*v1.V1TaskWithPayload, []*v1.DAGWithData, []v1.IdempotencyCollision, error) { if !p.optimisticSchedulingEnabled { - return nil, nil, nil, ErrNoOptimisticSlots + return nil, nil, nil, nil, ErrNoOptimisticSlots } // attempt to acquire a slot in the semaphore @@ -264,13 +264,13 @@ func (p *SchedulingPool) RunOptimisticScheduling(ctx context.Context, tenantId u }() default: // no slots available - return nil, nil, nil, ErrNoOptimisticSlots + return nil, nil, nil, nil, ErrNoOptimisticSlots } tm := p.getTenantManager(tenantId, false) if tm == nil { - return nil, nil, nil, ErrTenantNotFound + return nil, nil, nil, nil, ErrTenantNotFound } return tm.runOptimisticScheduling(ctx, opts, localWorkerIds) diff --git a/pkg/scheduling/v1/tenant_manager.go b/pkg/scheduling/v1/tenant_manager.go index cbf9a617d9..a9fa893528 100644 --- a/pkg/scheduling/v1/tenant_manager.go +++ b/pkg/scheduling/v1/tenant_manager.go @@ -430,21 +430,21 @@ func (t *tenantManager) runOptimisticScheduling( ctx context.Context, opts []*v1.WorkflowNameTriggerOpts, localWorkerIds map[uuid.UUID]struct{}, -) (map[uuid.UUID][]*AssignedItemWithTask, []*v1.V1TaskWithPayload, []*v1.DAGWithData, error) { +) (map[uuid.UUID][]*AssignedItemWithTask, []*v1.V1TaskWithPayload, []*v1.DAGWithData, []v1.IdempotencyCollision, error) { // create a transaction tx, err := t.cf.repo.Optimistic().StartTx(ctx) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } defer tx.Rollback() // hook into the trigger transaction - qis, tasks, dags, err := t.cf.repo.Optimistic().TriggerFromNames(ctx, tx, t.tenantId, opts) + qis, tasks, dags, collisions, err := t.cf.repo.Optimistic().TriggerFromNames(ctx, tx, t.tenantId, opts) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } // read the queue items for the tasks we just created @@ -470,7 +470,7 @@ func (t *tenantManager) runOptimisticScheduling( if err != nil { t.queuersMu.RUnlock() - return nil, nil, nil, err + return nil, nil, nil, nil, err } allLocalAssigned = append(allLocalAssigned, localAssigned...) @@ -481,7 +481,7 @@ func (t *tenantManager) runOptimisticScheduling( } if err := tx.Commit(ctx); err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } for _, qr := range allQueueResults { @@ -518,7 +518,7 @@ func (t *tenantManager) runOptimisticScheduling( }) } - return res, tasks, dags, nil + return res, tasks, dags, collisions, nil } func (t *tenantManager) runOptimisticSchedulingFromEvents( diff --git a/sdks/go/e2e/idempotency_test.go b/sdks/go/e2e/idempotency_test.go new file mode 100644 index 0000000000..8cfa5e48b7 --- /dev/null +++ b/sdks/go/e2e/idempotency_test.go @@ -0,0 +1,288 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + legacyclient "github.com/hatchet-dev/hatchet/pkg/client" + "github.com/hatchet-dev/hatchet/pkg/client/rest" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + "github.com/stretchr/testify/require" +) + +const ( + idempotencyEventKey = "go-e2e:idempotency-example" + idempotentTaskName = "go-e2e-idempotent-task" + idempotentShortWindowName = "go-e2e-idempotent-task-short-window" +) + +type idempotencyInput struct { + ID string `json:"id"` +} + +var ( + idempotencySetupOnce sync.Once + idempotencySetupErr error + idempotencyLegacyClient legacyclient.Client + idempotentTask *hatchet.StandaloneTask + idempotentTaskShortWindow *hatchet.StandaloneTask +) + +func setupIdempotencyWorker(t *testing.T) (legacyclient.Client, *hatchet.StandaloneTask, *hatchet.StandaloneTask) { + t.Helper() + + idempotencySetupOnce.Do(func() { + idempotencyLegacyClient, idempotencySetupErr = legacyclient.New() + if idempotencySetupErr != nil { + return + } + + idempotentTask = sharedClient.NewStandaloneTask( + idempotentTaskName, + func(ctx hatchet.Context, input idempotencyInput) (map[string]string, error) { + return map[string]string{"result": fmt.Sprintf("Hello, world from task %s", input.ID)}, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: time.Minute, + }), + hatchet.WithWorkflowEvents(idempotencyEventKey), + ) + + idempotentTaskShortWindow = sharedClient.NewStandaloneTask( + idempotentShortWindowName, + func(ctx hatchet.Context, input idempotencyInput) (map[string]string, error) { + return map[string]string{"result": fmt.Sprintf("Hello, world from task %s", input.ID)}, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: 2 * time.Second, + }), + ) + + worker, err := sharedClient.NewWorker( + "e2e-idempotency-worker", + hatchet.WithWorkflows(idempotentTask, idempotentTaskShortWindow), + ) + if err != nil { + idempotencySetupErr = err + return + } + + _, idempotencySetupErr = worker.Start() + if idempotencySetupErr != nil { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + + pollUntil(t, ctx, func() (bool, error) { + if _, err := sharedClient.Workflows().Get(ctx, idempotentTask.GetName()); err != nil { + return false, nil + } + if _, err := sharedClient.Workflows().Get(ctx, idempotentTaskShortWindow.GetName()); err != nil { + return false, nil + } + + return true, nil + }) + }) + + require.NoError(t, idempotencySetupErr) + + return idempotencyLegacyClient, idempotentTask, idempotentTaskShortWindow +} + +func listRunsByTestRunID(ctx context.Context, testRunID string) (*rest.V1TaskSummaryList, error) { + limit := int64(20) + metadata := []string{"test_run_id:" + testRunID} + + return sharedClient.Runs().List(ctx, rest.V1WorkflowRunListParams{ + Since: time.Now().Add(-5 * time.Minute), + Limit: &limit, + AdditionalMetadata: &metadata, + OnlyTasks: false, + }) +} + +func listEventsByTestRunID(ctx context.Context, legacy legacyclient.Client, testRunID string) (*rest.V1EventList, error) { + limit := int64(10) + metadata := []string{"test_run_id:" + testRunID} + + resp, err := legacy.API().V1EventListWithResponse( + ctx, + uuid.MustParse(legacy.TenantId()), + &rest.V1EventListParams{ + Since: &[]time.Time{time.Now().Add(-5 * time.Minute)}[0], + Limit: &limit, + AdditionalMetadata: &metadata, + }, + ) + if err != nil { + return nil, err + } + if resp.JSON200 == nil { + return nil, fmt.Errorf("unexpected event list status %d", resp.StatusCode()) + } + + return resp.JSON200, nil +} + +func TestIdempotencyDirectTrigger(t *testing.T) { + _, idempotentTask, _ := setupIdempotencyWorker(t) + ctx := newTestContext(t) + testRunID := uniqueID() + + ref1, err := idempotentTask.RunNoWait( + ctx, + idempotencyInput{ID: testRunID}, + hatchet.WithRunMetadata(map[string]string{"test_run_id": testRunID}), + ) + require.NoError(t, err) + + _, err = idempotentTask.RunNoWait(ctx, idempotencyInput{ID: testRunID}) + idempErr, ok := hatchet.IsIdempotencyCollisionError(err) + require.True(t, ok) + require.Equal(t, ref1.RunId, idempErr.ExistingRunExternalId) + + var runs *rest.V1TaskSummaryList + pollUntil(t, ctx, func() (bool, error) { + var pollErr error + runs, pollErr = listRunsByTestRunID(ctx, testRunID) + if pollErr != nil { + return false, pollErr + } + return len(runs.Rows) == 1, nil + }) + + require.NotNil(t, runs) + require.Len(t, runs.Rows, 1) + require.Equal(t, ref1.RunId, runs.Rows[0].Metadata.Id) +} + +func TestIdempotencyShortWindow(t *testing.T) { + _, _, idempotentTaskShortWindow := setupIdempotencyWorker(t) + ctx := newTestContext(t) + testRunID := uniqueID() + + for i := 0; i < 4; i++ { + if i == 1 { + _, err := idempotentTaskShortWindow.RunNoWait( + ctx, + idempotencyInput{ID: testRunID}, + hatchet.WithRunMetadata(map[string]string{"test_run_id": testRunID}), + ) + idempErr, ok := hatchet.IsIdempotencyCollisionError(err) + require.True(t, ok) + require.NotEmpty(t, idempErr.ExistingRunExternalId) + } else { + _, err := idempotentTaskShortWindow.RunNoWait( + ctx, + idempotencyInput{ID: testRunID}, + hatchet.WithRunMetadata(map[string]string{"test_run_id": testRunID}), + ) + require.NoError(t, err) + } + + if i != 3 { + time.Sleep(time.Duration(float64(i)*float64(time.Second) + 1.5*float64(time.Second))) + } + } + + var runs *rest.V1TaskSummaryList + pollUntil(t, ctx, func() (bool, error) { + var pollErr error + runs, pollErr = listRunsByTestRunID(ctx, testRunID) + if pollErr != nil { + return false, pollErr + } + return len(runs.Rows) >= 3, nil + }) + + require.NotNil(t, runs) + require.Len(t, runs.Rows, 3) +} + +func TestIdempotencyEventTrigger(t *testing.T) { + legacy, _, _ := setupIdempotencyWorker(t) + ctx := newTestContext(t) + testRunID := uniqueID() + + err := sharedClient.Events().Push( + ctx, + idempotencyEventKey, + idempotencyInput{ID: testRunID}, + legacyclient.WithEventMetadata(map[string]string{"test_run_id": testRunID}), + ) + require.NoError(t, err) + err = sharedClient.Events().Push( + ctx, + idempotencyEventKey, + idempotencyInput{ID: testRunID}, + legacyclient.WithEventMetadata(map[string]string{"test_run_id": testRunID}), + ) + require.NoError(t, err) + + var runs *rest.V1TaskSummaryList + pollUntil(t, ctx, func() (bool, error) { + var pollErr error + runs, pollErr = listRunsByTestRunID(ctx, testRunID) + if pollErr != nil { + return false, pollErr + } + return len(runs.Rows) == 1, nil + }) + + require.NotNil(t, runs) + require.Len(t, runs.Rows, 1) + + var events *rest.V1EventList + pollUntil(t, ctx, func() (bool, error) { + var pollErr error + events, pollErr = listEventsByTestRunID(ctx, legacy, testRunID) + if pollErr != nil { + return false, pollErr + } + return events.Rows != nil && len(*events.Rows) == 2, nil + }) + + require.NotNil(t, events) + require.Len(t, *events.Rows, 2) + + triggeredRunIDs := make(map[string]struct{}) + for _, event := range *events.Rows { + for _, triggeredRun := range derefTriggeredRuns(event.TriggeredRuns) { + triggeredRunIDs[triggeredRun.WorkflowRunId.String()] = struct{}{} + } + } + + require.Len(t, triggeredRunIDs, 1) + + var triggeredRunID string + for runID := range triggeredRunIDs { + triggeredRunID = runID + } + + pollUntil(t, ctx, func() (bool, error) { + status, pollErr := sharedClient.Runs().GetStatus(ctx, triggeredRunID) + if pollErr != nil { + return false, pollErr + } + return *status == rest.V1TaskStatusCOMPLETED, nil + }) +} + +func derefTriggeredRuns(triggeredRuns *[]rest.V1EventTriggeredRun) []rest.V1EventTriggeredRun { + if triggeredRuns == nil { + return nil + } + + return *triggeredRuns +} diff --git a/sdks/go/errors.go b/sdks/go/errors.go index 5cc132a463..2d263acf65 100644 --- a/sdks/go/errors.go +++ b/sdks/go/errors.go @@ -51,3 +51,21 @@ func IsEvictionNotSupportedError(err error) (*EvictionNotSupportedError, bool) { } return nil, false } + +// IdempotencyCollisionError is returned when an idempotency key collision occurs. +// It contains the ID of the existing run that claimed the key. +type IdempotencyCollisionError struct { + ExistingRunExternalId string +} + +func (e *IdempotencyCollisionError) Error() string { + return fmt.Sprintf("idempotency key collision: existing run %s already exists", e.ExistingRunExternalId) +} + +// IsIdempotencyCollisionError checks if the error is an IdempotencyCollisionError. +func IsIdempotencyCollisionError(err error) (*IdempotencyCollisionError, bool) { + if e, ok := err.(*IdempotencyCollisionError); ok { + return e, true + } + return nil, false +} diff --git a/sdks/go/examples/idempotency/trigger.go b/sdks/go/examples/idempotency/trigger.go new file mode 100644 index 0000000000..2afe30fa42 --- /dev/null +++ b/sdks/go/examples/idempotency/trigger.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create hatchet client: %v", err) + } + + idempotentTask := IdempotentTask(client) + + worker, err := client.NewWorker("idempotency-worker", hatchet.WithWorkflows(idempotentTask)) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + go func() { + if err := worker.StartBlocking(interruptCtx); err != nil { + log.Fatalf("failed to start worker: %v", err) + } + }() + + ctx := context.Background() + + // > trigger + ref1, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + if err != nil { + log.Fatalf("unexpected error on first run: %v", err) + } + + ref2, err := idempotentTask.RunNoWait(ctx, IdempotencyInput{ID: "123"}) + + var runID2 string + + if err != nil { + if idempErr, ok := hatchet.IsIdempotencyCollisionError(err); ok { + fmt.Printf("Run %s already exists for this idempotency key\n", idempErr.ExistingRunExternalId) + runID2 = idempErr.ExistingRunExternalId + } else { + log.Fatalf("unexpected error on second run: %v", err) + } + } else { + runID2 = ref2.RunId + } + // !! + + fmt.Printf("First run ID: %s\n", ref1.RunId) + fmt.Printf("Second run ID: %s\n", runID2) +} diff --git a/sdks/go/examples/idempotency/worker.go b/sdks/go/examples/idempotency/worker.go new file mode 100644 index 0000000000..030e6af6c0 --- /dev/null +++ b/sdks/go/examples/idempotency/worker.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "time" + + hatchet "github.com/hatchet-dev/hatchet/sdks/go" +) + +type IdempotencyInput struct { + ID string `json:"id"` +} + +type IdempotencyOutput struct { + Result string `json:"result"` +} + +// > idempotency +func IdempotentTask(client *hatchet.Client) *hatchet.StandaloneTask { + return client.NewStandaloneTask( + "idempotent-task", + func(ctx hatchet.Context, input IdempotencyInput) (*IdempotencyOutput, error) { + return &IdempotencyOutput{ + Result: fmt.Sprintf("Hello, world from task %s", input.ID), + }, nil + }, + hatchet.WithWorkflowIdempotency(hatchet.IdempotencyConfig{ + Expression: "input.id", + TTL: time.Minute, + }), + ) +} + +// !! diff --git a/sdks/go/internal/declaration.go b/sdks/go/internal/declaration.go index 0665298bcf..6b43dbb342 100644 --- a/sdks/go/internal/declaration.go +++ b/sdks/go/internal/declaration.go @@ -142,6 +142,7 @@ type workflowDeclarationImpl[I any, O any] struct { DefaultPriority *int32 DefaultFilters []types.DefaultFilter + Idempotency *create.IdempotencyConfig } // NewWorkflowDeclaration creates a new workflow declaration with the specified options and client. @@ -197,6 +198,7 @@ func NewWorkflowDeclaration[I any, O any](opts create.WorkflowCreateOpts[I], v0 outputSetters: make(map[string]func(*O, interface{})), DefaultPriority: opts.DefaultPriority, DefaultFilters: opts.DefaultFilters, + Idempotency: opts.Idempotency, } if opts.Version != "" { @@ -658,6 +660,13 @@ func (w *workflowDeclarationImpl[I, O]) Dump() (*contracts.CreateWorkflowVersion req.Sticky = &stickyStrategy } + if w.Idempotency != nil { + req.Idempotency = &contracts.IdempotencyConfig{ + Expression: w.Idempotency.Expression, + TtlMs: w.Idempotency.TTL.Milliseconds(), + } + } + // Create named function objects for regular tasks regularNamedFns := make([]NamedFunction, len(w.tasks)) for i, task := range w.tasks { diff --git a/sdks/go/workflow.go b/sdks/go/workflow.go index c018f95195..8dd1259f3d 100644 --- a/sdks/go/workflow.go +++ b/sdks/go/workflow.go @@ -148,6 +148,17 @@ func (w *Workflow) GetName() string { // WorkflowOption configures a workflow instance. type WorkflowOption func(*workflowConfig) +// IdempotencyConfig configures idempotency behavior for a workflow or standalone task. +// When set, runs triggered with the same computed key within the TTL window return an +// IdempotencyCollisionError instead of creating a new run. +type IdempotencyConfig struct { + // Expression is a CEL expression evaluated against the workflow input to produce an idempotency key. + Expression string + + // TTL is the duration during which duplicate runs with the same key are rejected. + TTL time.Duration +} + type workflowConfig struct { onCron []string onEvents []string @@ -159,6 +170,7 @@ type workflowConfig struct { stickyStrategy *types.StickyStrategy cronInput *string defaultFilters []types.DefaultFilter + idempotency *IdempotencyConfig } // WithWorkflowCron configures the workflow to run on a cron schedule. @@ -236,6 +248,15 @@ func WithWorkflowStickyStrategy(stickyStrategy types.StickyStrategy) WorkflowOpt } } +// WithWorkflowIdempotency configures idempotency for the workflow. +// When set, runs triggered with the same computed key within the TTL window return an +// IdempotencyCollisionError instead of creating a new run. +func WithWorkflowIdempotency(config IdempotencyConfig) WorkflowOption { + return func(c *workflowConfig) { + c.idempotency = &config + } +} + // newWorkflow creates a new workflow definition. func newWorkflow(name string, v0Client v0Client.Client, options ...WorkflowOption) *Workflow { config := &workflowConfig{} @@ -267,6 +288,13 @@ func newWorkflow(name string, v0Client v0Client.Client, options ...WorkflowOptio createOpts.DefaultPriority = &priority } + if config.idempotency != nil { + createOpts.Idempotency = &create.IdempotencyConfig{ + Expression: config.idempotency.Expression, + TTL: config.idempotency.TTL, + } + } + declaration := internal.NewWorkflowDeclaration[any, any](createOpts, v0Client) return &Workflow{ @@ -726,6 +754,14 @@ func (w *Workflow) runWorkflowInternal(ctx context.Context, otelCtx context.Cont } if err != nil { + var idempViolation *v0Client.IdempotencyViolationErr + if errors.As(err, &idempViolation) { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + return nil, &IdempotencyCollisionError{ + ExistingRunExternalId: idempViolation.ExistingRunExternalId, + } + } span.SetStatus(codes.Error, err.Error()) span.RecordError(err) return nil, err diff --git a/sdks/python/CHANGELOG.md b/sdks/python/CHANGELOG.md index 7fba72574e..083e16d002 100644 --- a/sdks/python/CHANGELOG.md +++ b/sdks/python/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Hatchet's Python SDK will be documented in this changelog The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.34.0] + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks, which ensures that they're only run once in a provided time window, based on a CEL expression. + ## [1.33.17] - 2026-07-07 ### Fixed diff --git a/sdks/python/examples/events/test_event.py b/sdks/python/examples/events/test_event.py index 32b5fa1100..1e5cd03296 100644 --- a/sdks/python/examples/events/test_event.py +++ b/sdks/python/examples/events/test_event.py @@ -605,6 +605,7 @@ async def test_multi_scope_bug(hatchet: Hatchet, test_run_id: str) -> None: runs = await hatchet.runs.aio_list( triggering_event_external_id=event.event_id, additional_metadata={"test_run_id": test_run_id}, + workflow_ids=[event_workflow.id], ) assert len(runs.rows) == 1 diff --git a/sdks/python/examples/idempotency/test_idempotency.py b/sdks/python/examples/idempotency/test_idempotency.py new file mode 100644 index 0000000000..a795a09922 --- /dev/null +++ b/sdks/python/examples/idempotency/test_idempotency.py @@ -0,0 +1,156 @@ +import pytest + +from examples.idempotency.worker import ( + idempotent_task, + idempotent_task_short_window, + IdempotencyInput, + EVENT_KEY, +) + +from hatchet_sdk import Hatchet, IdempotencyCollisionError, RunStatus +from hatchet_sdk.clients.rest.models.v1_task_summary_list import V1TaskSummaryList +from uuid import uuid4 +from datetime import timedelta, datetime, timezone +import asyncio + + +@pytest.mark.asyncio(loop_scope="session") +async def test_idempotency_keys_prevent_duplicate_runs_direct_trigger( + hatchet: Hatchet, +) -> None: + test_run_id = str(uuid4()) + ref1 = await idempotent_task.aio_run( + input=IdempotencyInput(id=test_run_id), + wait_for_result=False, + additional_metadata={"test_run_id": test_run_id}, + ) + + with pytest.raises(IdempotencyCollisionError) as exc_info: + await idempotent_task.aio_run( + input=IdempotencyInput(id=test_run_id), wait_for_result=False + ) + + assert exc_info.value.existing_run_external_id == ref1.workflow_run_id + + runs: V1TaskSummaryList | None = None + + for _ in range(15): + runs = await hatchet.runs.aio_list( + since=datetime.now(timezone.utc) - timedelta(minutes=5), + additional_metadata={"test_run_id": test_run_id}, + ) + + if len(runs.rows) == 0: + await asyncio.sleep(1) + continue + + break + + assert runs is not None + assert len(runs.rows) == 1 + assert runs.rows[0].metadata.id == ref1.workflow_run_id + + +@pytest.mark.asyncio(loop_scope="session") +async def test_idempotency_keys_prevent_duplicate_runs_direct_trigger_short_window( + hatchet: Hatchet, +) -> None: + test_run_id = str(uuid4()) + for i in range(4): + if i == 1: + with pytest.raises(IdempotencyCollisionError) as exc_info: + await idempotent_task_short_window.aio_run( + input=IdempotencyInput(id=test_run_id), + wait_for_result=False, + additional_metadata={"test_run_id": test_run_id}, + ) + + assert exc_info.value.existing_run_external_id is not None + else: + await idempotent_task_short_window.aio_run( + input=IdempotencyInput(id=test_run_id), + wait_for_result=False, + additional_metadata={"test_run_id": test_run_id}, + ) + + ## dynamic sleep, first task should run, second should not, third should, fourth should + if i != 3: + await asyncio.sleep(i + 1.5) + + runs: V1TaskSummaryList | None = None + + for _ in range(15): + runs = await hatchet.runs.aio_list( + since=datetime.now(timezone.utc) - timedelta(minutes=5), + additional_metadata={"test_run_id": test_run_id}, + ) + + if len(runs.rows) < 3: + await asyncio.sleep(1) + continue + + break + else: + pytest.fail("Expected to find at least one run, but found none.") + + assert runs.rows + assert len(runs.rows) == 3 + + +@pytest.mark.asyncio(loop_scope="session") +async def test_idempotency_keys_prevent_duplicate_runs_event_trigger( + hatchet: Hatchet, +) -> None: + test_run_id = str(uuid4()) + e1 = await hatchet.event.aio_push( + event_key=EVENT_KEY, + payload={"id": test_run_id}, + additional_metadata={"test_run_id": test_run_id}, + ) + e2 = await hatchet.event.aio_push( + event_key=EVENT_KEY, + payload={"id": test_run_id}, + additional_metadata={"test_run_id": test_run_id}, + ) + + runs: V1TaskSummaryList | None = None + + for _ in range(15): + runs = await hatchet.runs.aio_list( + since=datetime.now(timezone.utc) - timedelta(minutes=5), + additional_metadata={"test_run_id": test_run_id}, + ) + + if len(runs.rows) == 0: + await asyncio.sleep(1) + continue + + break + + assert runs is not None + assert len(runs.rows) == 1 + + details = await hatchet.event.aio_list( + event_ids=[e1.event_id, e2.event_id], + ) + + assert details.rows + assert len(details.rows) == 2 + + all_triggered_runs = [ + *(details.rows[0].triggered_runs or []), + *(details.rows[1].triggered_runs or []), + ] + + assert len(all_triggered_runs) == 1 + + for _ in range(15): + run_details = await hatchet.runs.aio_get_details( + all_triggered_runs[0].workflow_run_id + ) + + if run_details.status in [RunStatus.QUEUED, RunStatus.RUNNING]: + await asyncio.sleep(1) + continue + + assert run_details.status == RunStatus.COMPLETED diff --git a/sdks/python/examples/idempotency/trigger.py b/sdks/python/examples/idempotency/trigger.py new file mode 100644 index 0000000000..aacf8416fd --- /dev/null +++ b/sdks/python/examples/idempotency/trigger.py @@ -0,0 +1,32 @@ +import asyncio + +from hatchet_sdk import IdempotencyCollisionError + +from examples.idempotency.worker import idempotent_task, IdempotencyInput + + +async def main() -> None: + # > trigger + ref_1 = await idempotent_task.aio_run( + input=IdempotencyInput(id="123"), + wait_for_result=False, + ) + + try: + ref_2 = await idempotent_task.aio_run( + input=IdempotencyInput(id="123"), + wait_for_result=False, + ) + run_id_2 = ref_2.workflow_run_id + except IdempotencyCollisionError as e: + print( + f"Run with external ID {e.existing_run_external_id} already exists for this idempotency key" + ) + run_id_2 = e.existing_run_external_id + + res_1 = await ref_1.aio_result() + res_2 = await idempotent_task.aio_get_result(run_id_2) + + assert res_1 == res_2 + assert ref_1.workflow_run_id == run_id_2 + # !! diff --git a/sdks/python/examples/idempotency/worker.py b/sdks/python/examples/idempotency/worker.py new file mode 100644 index 0000000000..ab647d85fa --- /dev/null +++ b/sdks/python/examples/idempotency/worker.py @@ -0,0 +1,48 @@ +from hatchet_sdk import Context, Hatchet, IdempotencyConfig +from datetime import timedelta +from pydantic import BaseModel + +hatchet = Hatchet() + +# > idempotency + +EVENT_KEY = "idempotency:example" + + +class IdempotencyInput(BaseModel): + id: str + + +@hatchet.task( + idempotency=IdempotencyConfig(key_expression="input.id", ttl=timedelta(minutes=1)), + input_validator=IdempotencyInput, + on_events=[EVENT_KEY], +) +async def idempotent_task(input: IdempotencyInput, ctx: Context) -> dict[str, str]: + return {"result": f"Hello, world from task {input.id}"} + + +# !! + + +@hatchet.task( + idempotency=IdempotencyConfig(key_expression="input.id", ttl=timedelta(seconds=2)), + input_validator=IdempotencyInput, + on_events=[EVENT_KEY], +) +async def idempotent_task_short_window( + input: IdempotencyInput, ctx: Context +) -> dict[str, str]: + return {"result": f"Hello, world from task {input.id}"} + + +def main() -> None: + worker = hatchet.worker( + "test-worker", + workflows=[idempotent_task], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/sdks/python/examples/worker.py b/sdks/python/examples/worker.py index 59f74cd52a..d4ced5e503 100644 --- a/sdks/python/examples/worker.py +++ b/sdks/python/examples/worker.py @@ -103,6 +103,7 @@ durable_parent_child_key_bug, child_child_key_bug, ) +from examples.idempotency.worker import idempotent_task, idempotent_task_short_window from examples.bug_tests.durable_spawn_index_collision.worker import ( durable_spawn_index_collision, spawn_index_child_a, @@ -205,6 +206,8 @@ def main() -> None: spawn_index_child_b, durable_child_key_dedup_replay, durable_spawn_many_dags, + idempotent_task, + idempotent_task_short_window, error_raising_durable_parent, error_raising_task, ], diff --git a/sdks/python/hatchet_sdk/__init__.py b/sdks/python/hatchet_sdk/__init__.py index 27e466bff9..9b481f8b26 100644 --- a/sdks/python/hatchet_sdk/__init__.py +++ b/sdks/python/hatchet_sdk/__init__.py @@ -147,6 +147,7 @@ DedupeViolationError, EvictionNotSupportedError, FailedTaskRunExceptionGroup, + IdempotencyCollisionError, NonDeterminismError, NonRetryableException, TaskRunError, @@ -167,6 +168,7 @@ ConcurrencyExpression, ConcurrencyLimitStrategy, ) +from hatchet_sdk.types.idempotency import IdempotencyConfig from hatchet_sdk.types.labels import ( DesiredWorkerLabel, WorkerLabel, @@ -236,6 +238,8 @@ "GithubBranch", "GithubRepo", "Hatchet", + "IdempotencyCollisionError", + "IdempotencyConfig", "Job", "JobRun", "JobRunStatus", diff --git a/sdks/python/hatchet_sdk/clients/admin.py b/sdks/python/hatchet_sdk/clients/admin.py index be15f0f3a5..f273e53762 100644 --- a/sdks/python/hatchet_sdk/clients/admin.py +++ b/sdks/python/hatchet_sdk/clients/admin.py @@ -3,10 +3,11 @@ from collections.abc import Generator from datetime import datetime from enum import Enum -from typing import TypeVar, cast +from typing import Any, TypeVar, cast import grpc from google.protobuf import timestamp_pb2 +from grpc_status import rpc_status from pydantic import BaseModel, ConfigDict, field_validator from hatchet_sdk.clients.listeners.run_event_listener import RunEventListenerClient @@ -20,7 +21,7 @@ from hatchet_sdk.contracts.v1.shared import trigger_pb2 as trigger_protos from hatchet_sdk.contracts.v1.workflows_pb2_grpc import AdminServiceStub from hatchet_sdk.contracts.workflows_pb2_grpc import WorkflowServiceStub -from hatchet_sdk.exceptions import DedupeViolationError +from hatchet_sdk.exceptions import DedupeViolationError, IdempotencyCollisionError from hatchet_sdk.logger import logger from hatchet_sdk.runnables.contextvars import ( ctx_action_key, @@ -151,6 +152,13 @@ def _get_or_create_v0_client(self) -> WorkflowServiceStub: return self.v0_client + def _get_or_create_v1_client(self) -> AdminServiceStub: + if self.client is None: + conn = new_conn(self.config, False) + self.client = AdminServiceStub(conn) + + return self.client + class TriggerWorkflowRequest(BaseModel): model_config = ConfigDict(extra="ignore") @@ -289,11 +297,8 @@ def put_workflow( self, workflow: workflow_protos.CreateWorkflowVersionRequest, ) -> workflow_protos.CreateWorkflowVersionResponse: - if self.client is None: - conn = new_conn(self.config, False) - self.client = AdminServiceStub(conn) - - put_workflow = tenacity_retry(self.client.PutWorkflow, self.config.tenacity) + client = self._get_or_create_v1_client() + put_workflow = tenacity_retry(client.PutWorkflow, self.config.tenacity) return cast( workflow_protos.CreateWorkflowVersionResponse, put_workflow( @@ -398,7 +403,6 @@ def _create_workflow_run_request( ) namespace = options.namespace or self.namespace - workflow_name = self.config.apply_namespace(workflow_name, namespace) return self._prepare_workflow_request(workflow_name, input, trigger_options) @@ -422,8 +426,23 @@ def run_workflow( ), ) except (grpc.RpcError, grpc.aio.AioRpcError) as e: - if e.code() == grpc.StatusCode.ALREADY_EXISTS: + if ( + e.code() == grpc.StatusCode.ALREADY_EXISTS + and e.details() == "idempotency key collision" + ): + status: list[Any] = rpc_status.from_call(e) # type: ignore[arg-type] + + for detail in status.details: # type: ignore[attr-defined] + if detail.Is(workflow_protos.IdempotencyCollisionError.DESCRIPTOR): + info = workflow_protos.IdempotencyCollisionError() + detail.Unpack(info) + + raise IdempotencyCollisionError( + existing_run_external_id=info.existing_run_external_id + ) from e + elif e.code() == grpc.StatusCode.ALREADY_EXISTS: raise DedupeViolationError(e.details()) from e + raise e return WorkflowRunRef( @@ -439,11 +458,12 @@ async def aio_run_workflow( input: str | None, options: TriggerWorkflowOptions = TriggerWorkflowOptions(), ) -> WorkflowRunRef: - client = self._get_or_create_v0_client() - trigger_workflow = tenacity_retry(client.TriggerWorkflow, self.config.tenacity) async with spawn_index_lock: request = self._create_workflow_run_request(workflow_name, input, options) + client = self._get_or_create_v0_client() + trigger_workflow = tenacity_retry(client.TriggerWorkflow, self.config.tenacity) + try: resp = cast( v0_workflow_protos.TriggerWorkflowResponse, @@ -454,7 +474,21 @@ async def aio_run_workflow( ), ) except (grpc.RpcError, grpc.aio.AioRpcError) as e: - if e.code() == grpc.StatusCode.ALREADY_EXISTS: + if ( + e.code() == grpc.StatusCode.ALREADY_EXISTS + and e.details() == "idempotency key collision" + ): + status: list[Any] = rpc_status.from_call(e) # type: ignore[arg-type] + + for detail in status.details: # type: ignore[attr-defined] + if detail.Is(workflow_protos.IdempotencyCollisionError.DESCRIPTOR): + info = workflow_protos.IdempotencyCollisionError() + detail.Unpack(info) + + raise IdempotencyCollisionError( + existing_run_external_id=info.existing_run_external_id + ) from e + elif e.code() == grpc.StatusCode.ALREADY_EXISTS: raise DedupeViolationError(e.details()) from e raise e @@ -590,12 +624,10 @@ def get_workflow_run(self, workflow_run_id: str) -> WorkflowRunRef: ) def get_details(self, external_id: str) -> WorkflowRunDetail: - if self.client is None: - conn = new_conn(self.config, False) - self.client = AdminServiceStub(conn) - + client = self._get_or_create_v1_client() get_run_payloads = tenacity_retry( - self.client.GetRunDetails, self.config.tenacity + client.GetRunDetails, + self.config.tenacity.model_copy(update={"retry_not_found": True}), ) response = cast( diff --git a/sdks/python/hatchet_sdk/clients/rest/tenacity_utils.py b/sdks/python/hatchet_sdk/clients/rest/tenacity_utils.py index 7a543a19ea..2add2d11cb 100644 --- a/sdks/python/hatchet_sdk/clients/rest/tenacity_utils.py +++ b/sdks/python/hatchet_sdk/clients/rest/tenacity_utils.py @@ -40,22 +40,28 @@ def tenacity_should_retry( ex: BaseException, config: TenacityConfig | None = None ) -> bool: """Return True when the exception should be retried.""" - if isinstance(ex, ServiceException | NotFoundException): + if isinstance(ex, (ServiceException, NotFoundException)): return True if isinstance(ex, TooManyRequestsException): return bool(config and config.retry_429) # gRPC errors: retry most, except specific permanent failure codes - if isinstance(ex, grpc.aio.AioRpcError | grpc.RpcError): - return ex.code() not in [ + if isinstance(ex, (grpc.aio.AioRpcError, grpc.RpcError)): + non_retryable = [ grpc.StatusCode.UNIMPLEMENTED, - grpc.StatusCode.NOT_FOUND, grpc.StatusCode.INVALID_ARGUMENT, grpc.StatusCode.ALREADY_EXISTS, grpc.StatusCode.UNAUTHENTICATED, grpc.StatusCode.PERMISSION_DENIED, ] + if not config or not config.retry_not_found: + ## don't retry NOT_FOUND by default, + ## but allow it to be configurable so that we can + ## allow this internally, e.g. in `get_details` + non_retryable.append(grpc.StatusCode.NOT_FOUND) + + return ex.code() not in non_retryable # REST transport errors: opt-in retry for configured HTTP methods if isinstance(ex, RestTransportError): diff --git a/sdks/python/hatchet_sdk/config.py b/sdks/python/hatchet_sdk/config.py index ca12f34c8a..e11f71b744 100644 --- a/sdks/python/hatchet_sdk/config.py +++ b/sdks/python/hatchet_sdk/config.py @@ -129,6 +129,10 @@ class TenacityConfig(BaseSettings): default=False, description="Enable retries for HTTP 429 Too Many Requests responses. Default: off.", ) + retry_not_found: bool = Field( + default=False, + description="Enable retries for NOT_FOUND gRPC responses. For resources that may not yet exist due to async creation. Default: off.", + ) retry_transport_errors: bool = Field( default=False, description="Enable retries for REST transport errors (timeout, connection, TLS). Default: off.", diff --git a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py index 3a9ab133c1..9ec0c18f81 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py +++ b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py @@ -27,7 +27,7 @@ from hatchet_sdk.contracts.v1.shared import trigger_pb2 as v1_dot_shared_dot_trigger__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xae\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priority\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xac\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schema\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xae\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priority\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xed\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x12/\n\x0bidempotency\x18\x0f \x01(\x0b\x32\x15.v1.IdempotencyConfigH\x05\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schemaB\x0e\n\x0c_idempotency\"7\n\x11IdempotencyConfig\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x0e\n\x06ttl_ms\x18\x02 \x01(\x03\"=\n\x19IdempotencyCollisionError\x12 \n\x18\x65xisting_run_external_id\x18\x01 \x01(\t\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -43,14 +43,14 @@ _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_options = b'8\001' _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._loaded_options = None _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_options = b'8\001' - _globals['_STICKYSTRATEGY']._serialized_start=3411 - _globals['_STICKYSTRATEGY']._serialized_end=3447 - _globals['_RATELIMITDURATION']._serialized_start=3449 - _globals['_RATELIMITDURATION']._serialized_end=3542 - _globals['_RUNSTATUS']._serialized_start=3544 - _globals['_RUNSTATUS']._serialized_end=3635 - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=3637 - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=3764 + _globals['_STICKYSTRATEGY']._serialized_start=3596 + _globals['_STICKYSTRATEGY']._serialized_end=3632 + _globals['_RATELIMITDURATION']._serialized_start=3634 + _globals['_RATELIMITDURATION']._serialized_end=3727 + _globals['_RUNSTATUS']._serialized_start=3729 + _globals['_RUNSTATUS']._serialized_end=3820 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=3822 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=3949 _globals['_CANCELTASKSREQUEST']._serialized_start=111 _globals['_CANCELTASKSREQUEST']._serialized_end=202 _globals['_REPLAYTASKSREQUEST']._serialized_start=204 @@ -72,29 +72,33 @@ _globals['_BRANCHDURABLETASKRESPONSE']._serialized_start=1024 _globals['_BRANCHDURABLETASKRESPONSE']._serialized_end=1113 _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_start=1116 - _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_end=1672 - _globals['_DEFAULTFILTER']._serialized_start=1674 - _globals['_DEFAULTFILTER']._serialized_end=1758 - _globals['_CONCURRENCY']._serialized_start=1761 - _globals['_CONCURRENCY']._serialized_end=1908 - _globals['_CREATETASKOPTS']._serialized_start=1911 - _globals['_CREATETASKOPTS']._serialized_end=2606 - _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_start=2398 - _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_end=2474 - _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_start=2476 - _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_end=2527 - _globals['_CREATETASKRATELIMIT']._serialized_start=2609 - _globals['_CREATETASKRATELIMIT']._serialized_end=2862 - _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_start=2864 - _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_end=2928 - _globals['_GETRUNDETAILSREQUEST']._serialized_start=2930 - _globals['_GETRUNDETAILSREQUEST']._serialized_end=2973 - _globals['_TASKRUNDETAIL']._serialized_start=2976 - _globals['_TASKRUNDETAIL']._serialized_end=3146 - _globals['_GETRUNDETAILSRESPONSE']._serialized_start=3149 - _globals['_GETRUNDETAILSRESPONSE']._serialized_end=3409 - _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_start=3343 - _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_end=3409 - _globals['_ADMINSERVICE']._serialized_start=3767 - _globals['_ADMINSERVICE']._serialized_end=4230 + _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_end=1737 + _globals['_IDEMPOTENCYCONFIG']._serialized_start=1739 + _globals['_IDEMPOTENCYCONFIG']._serialized_end=1794 + _globals['_IDEMPOTENCYCOLLISIONERROR']._serialized_start=1796 + _globals['_IDEMPOTENCYCOLLISIONERROR']._serialized_end=1857 + _globals['_DEFAULTFILTER']._serialized_start=1859 + _globals['_DEFAULTFILTER']._serialized_end=1943 + _globals['_CONCURRENCY']._serialized_start=1946 + _globals['_CONCURRENCY']._serialized_end=2093 + _globals['_CREATETASKOPTS']._serialized_start=2096 + _globals['_CREATETASKOPTS']._serialized_end=2791 + _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_start=2583 + _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_end=2659 + _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_start=2661 + _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_end=2712 + _globals['_CREATETASKRATELIMIT']._serialized_start=2794 + _globals['_CREATETASKRATELIMIT']._serialized_end=3047 + _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_start=3049 + _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_end=3113 + _globals['_GETRUNDETAILSREQUEST']._serialized_start=3115 + _globals['_GETRUNDETAILSREQUEST']._serialized_end=3158 + _globals['_TASKRUNDETAIL']._serialized_start=3161 + _globals['_TASKRUNDETAIL']._serialized_end=3331 + _globals['_GETRUNDETAILSRESPONSE']._serialized_start=3334 + _globals['_GETRUNDETAILSRESPONSE']._serialized_end=3594 + _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_start=3528 + _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_end=3594 + _globals['_ADMINSERVICE']._serialized_start=3952 + _globals['_ADMINSERVICE']._serialized_end=4415 # @@protoc_insertion_point(module_scope) diff --git a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi index 3015f82678..c262cb21a7 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi +++ b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi @@ -154,7 +154,7 @@ class BranchDurableTaskResponse(_message.Message): def __init__(self, task_external_id: _Optional[str] = ..., node_id: _Optional[int] = ..., branch_id: _Optional[int] = ...) -> None: ... class CreateWorkflowVersionRequest(_message.Message): - __slots__ = ("name", "description", "version", "event_triggers", "cron_triggers", "tasks", "concurrency", "cron_input", "on_failure_task", "sticky", "default_priority", "concurrency_arr", "default_filters", "input_json_schema") + __slots__ = ("name", "description", "version", "event_triggers", "cron_triggers", "tasks", "concurrency", "cron_input", "on_failure_task", "sticky", "default_priority", "concurrency_arr", "default_filters", "input_json_schema", "idempotency") NAME_FIELD_NUMBER: _ClassVar[int] DESCRIPTION_FIELD_NUMBER: _ClassVar[int] VERSION_FIELD_NUMBER: _ClassVar[int] @@ -169,6 +169,7 @@ class CreateWorkflowVersionRequest(_message.Message): CONCURRENCY_ARR_FIELD_NUMBER: _ClassVar[int] DEFAULT_FILTERS_FIELD_NUMBER: _ClassVar[int] INPUT_JSON_SCHEMA_FIELD_NUMBER: _ClassVar[int] + IDEMPOTENCY_FIELD_NUMBER: _ClassVar[int] name: str description: str version: str @@ -183,7 +184,22 @@ class CreateWorkflowVersionRequest(_message.Message): concurrency_arr: _containers.RepeatedCompositeFieldContainer[Concurrency] default_filters: _containers.RepeatedCompositeFieldContainer[DefaultFilter] input_json_schema: bytes - def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., version: _Optional[str] = ..., event_triggers: _Optional[_Iterable[str]] = ..., cron_triggers: _Optional[_Iterable[str]] = ..., tasks: _Optional[_Iterable[_Union[CreateTaskOpts, _Mapping]]] = ..., concurrency: _Optional[_Union[Concurrency, _Mapping]] = ..., cron_input: _Optional[str] = ..., on_failure_task: _Optional[_Union[CreateTaskOpts, _Mapping]] = ..., sticky: _Optional[_Union[StickyStrategy, str]] = ..., default_priority: _Optional[int] = ..., concurrency_arr: _Optional[_Iterable[_Union[Concurrency, _Mapping]]] = ..., default_filters: _Optional[_Iterable[_Union[DefaultFilter, _Mapping]]] = ..., input_json_schema: _Optional[bytes] = ...) -> None: ... + idempotency: IdempotencyConfig + def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., version: _Optional[str] = ..., event_triggers: _Optional[_Iterable[str]] = ..., cron_triggers: _Optional[_Iterable[str]] = ..., tasks: _Optional[_Iterable[_Union[CreateTaskOpts, _Mapping]]] = ..., concurrency: _Optional[_Union[Concurrency, _Mapping]] = ..., cron_input: _Optional[str] = ..., on_failure_task: _Optional[_Union[CreateTaskOpts, _Mapping]] = ..., sticky: _Optional[_Union[StickyStrategy, str]] = ..., default_priority: _Optional[int] = ..., concurrency_arr: _Optional[_Iterable[_Union[Concurrency, _Mapping]]] = ..., default_filters: _Optional[_Iterable[_Union[DefaultFilter, _Mapping]]] = ..., input_json_schema: _Optional[bytes] = ..., idempotency: _Optional[_Union[IdempotencyConfig, _Mapping]] = ...) -> None: ... + +class IdempotencyConfig(_message.Message): + __slots__ = ("expression", "ttl_ms") + EXPRESSION_FIELD_NUMBER: _ClassVar[int] + TTL_MS_FIELD_NUMBER: _ClassVar[int] + expression: str + ttl_ms: int + def __init__(self, expression: _Optional[str] = ..., ttl_ms: _Optional[int] = ...) -> None: ... + +class IdempotencyCollisionError(_message.Message): + __slots__ = ("existing_run_external_id",) + EXISTING_RUN_EXTERNAL_ID_FIELD_NUMBER: _ClassVar[int] + existing_run_external_id: str + def __init__(self, existing_run_external_id: _Optional[str] = ...) -> None: ... class DefaultFilter(_message.Message): __slots__ = ("expression", "scope", "payload") diff --git a/sdks/python/hatchet_sdk/exceptions.py b/sdks/python/hatchet_sdk/exceptions.py index 0b383da9b3..ff1b8e37bc 100644 --- a/sdks/python/hatchet_sdk/exceptions.py +++ b/sdks/python/hatchet_sdk/exceptions.py @@ -37,6 +37,12 @@ class DedupeViolationError(Exception): """Raised by the Hatchet library to indicate that a workflow has already been run with this deduplication value.""" +class IdempotencyCollisionError(Exception): + def __init__(self, existing_run_external_id: str) -> None: + super().__init__() + self.existing_run_external_id = existing_run_external_id + + TASK_RUN_ERROR_METADATA_KEY = "__hatchet_error_metadata__" diff --git a/sdks/python/hatchet_sdk/hatchet.py b/sdks/python/hatchet_sdk/hatchet.py index fd67366af0..94b135a483 100644 --- a/sdks/python/hatchet_sdk/hatchet.py +++ b/sdks/python/hatchet_sdk/hatchet.py @@ -41,6 +41,7 @@ ) from hatchet_sdk.runnables.workflow import BaseWorkflow, Standalone, Workflow from hatchet_sdk.types.concurrency import ConcurrencyExpression +from hatchet_sdk.types.idempotency import IdempotencyConfig from hatchet_sdk.types.labels import DesiredWorkerLabel from hatchet_sdk.types.priority import Priority, _warn_if_int_priority from hatchet_sdk.types.rate_limit import RateLimit @@ -278,6 +279,7 @@ def workflow( task_defaults: TaskDefaults = TaskDefaults(), default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> Workflow[EmptyModel]: ... @overload @@ -298,6 +300,7 @@ def workflow( task_defaults: TaskDefaults = TaskDefaults(), default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> Workflow[TWorkflowInput]: ... def workflow( @@ -317,6 +320,7 @@ def workflow( task_defaults: TaskDefaults = TaskDefaults(), default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> Workflow[EmptyModel] | Workflow[TWorkflowInput]: """ Define a Hatchet workflow, which can then declare `task`s and be `run`, `schedule`d, and so on. @@ -345,6 +349,8 @@ def workflow( :param default_additional_metadata: A dictionary of additional metadata to attach to each run of this workflow by default. + :param idempotency: An optional idempotency configuration for the workflow, controlling how Hatchet should determine if two runs of this workflow are "the same" for the purposes of deduplication and idempotent execution. + :returns: The created `Workflow` object, which can be used to declare tasks, run the workflow, and so on. """ @@ -364,6 +370,7 @@ def workflow( default_priority=default_priority, default_filters=default_filters or [], default_additional_metadata=default_additional_metadata or {}, + idempotency=idempotency, ), self, ) @@ -394,6 +401,7 @@ def task( backoff_max_seconds: int | None = None, default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> Callable[ [Callable[Concatenate[EmptyModel, Context, P], R | CoroutineLike[R]]], Standalone[EmptyModel, R], @@ -425,6 +433,7 @@ def task( backoff_max_seconds: int | None = None, default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> Callable[ [Callable[Concatenate[TWorkflowInput, Context, P], R | CoroutineLike[R]]], Standalone[TWorkflowInput, R], @@ -455,6 +464,7 @@ def task( backoff_max_seconds: int | None = None, default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, + idempotency: IdempotencyConfig | None = None, ) -> ( Callable[ [Callable[Concatenate[EmptyModel, Context, P], R | CoroutineLike[R]]], @@ -504,6 +514,8 @@ def task( :param default_additional_metadata: A dictionary of additional metadata to attach to each run of this task by default. + :param idempotency: An optional idempotency configuration for the task, controlling how Hatchet should determine if two runs of this task are "the same" for the purposes of deduplication and idempotent execution. + :returns: A decorator which creates a `Standalone` task object. """ @@ -528,6 +540,7 @@ def inner( input_validator=TypeAdapter(normalize_validator(input_validator)), default_filters=default_filters or [], default_additional_metadata=default_additional_metadata or {}, + idempotency=idempotency, ), self, ) @@ -590,6 +603,7 @@ def durable_task( default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, eviction_policy: EvictionPolicy | None = DEFAULT_DURABLE_TASK_EVICTION_POLICY, + idempotency: IdempotencyConfig | None = None, ) -> Callable[ [Callable[Concatenate[EmptyModel, DurableContext, P], R | CoroutineLike[R]]], Standalone[EmptyModel, R], @@ -622,6 +636,7 @@ def durable_task( default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, eviction_policy: EvictionPolicy | None = DEFAULT_DURABLE_TASK_EVICTION_POLICY, + idempotency: IdempotencyConfig | None = None, ) -> Callable[ [ Callable[ @@ -657,6 +672,7 @@ def durable_task( default_filters: list[DefaultFilter] | None = None, default_additional_metadata: JSONSerializableMapping | None = None, eviction_policy: EvictionPolicy | None = DEFAULT_DURABLE_TASK_EVICTION_POLICY, + idempotency: IdempotencyConfig | None = None, ) -> ( Callable[ [ @@ -716,6 +732,8 @@ def durable_task( :param eviction_policy: An optional eviction policy controlling when idle durable tasks are evicted from workers. + :param idempotency: An optional idempotency configuration for the task, controlling how Hatchet should determine if two runs of this task are "the same" for the purposes of deduplication and idempotent execution. + :returns: A decorator which creates a `Standalone` task object. """ @@ -737,6 +755,7 @@ def inner( default_priority=default_priority, default_filters=default_filters or [], default_additional_metadata=default_additional_metadata or {}, + idempotency=idempotency, ), self, ) diff --git a/sdks/python/hatchet_sdk/runnables/types.py b/sdks/python/hatchet_sdk/runnables/types.py index b6b393d93f..e0ea55bc55 100644 --- a/sdks/python/hatchet_sdk/runnables/types.py +++ b/sdks/python/hatchet_sdk/runnables/types.py @@ -18,6 +18,7 @@ from hatchet_sdk.types.concurrency import ( ConcurrencyExpression, ) +from hatchet_sdk.types.idempotency import IdempotencyConfig from hatchet_sdk.types.priority import Priority from hatchet_sdk.types.sticky import StickyStrategy from hatchet_sdk.utils.timedelta_to_expression import Duration @@ -100,6 +101,7 @@ class WorkflowConfig(BaseModel): concurrency: int | ConcurrencyExpression | list[ConcurrencyExpression] | None = None input_validator: TypeAdapter[TaskPayloadForInternalUse] default_priority: int | Priority | None = None + idempotency: IdempotencyConfig | None = None task_defaults: TaskDefaults = TaskDefaults() default_filters: list[DefaultFilter] = Field(default_factory=list) diff --git a/sdks/python/hatchet_sdk/runnables/workflow.py b/sdks/python/hatchet_sdk/runnables/workflow.py index 56966f5faf..da9eae1c3a 100644 --- a/sdks/python/hatchet_sdk/runnables/workflow.py +++ b/sdks/python/hatchet_sdk/runnables/workflow.py @@ -264,6 +264,11 @@ def to_proto(self) -> CreateWorkflowVersionRequest: default_priority=self._config.default_priority, default_filters=[f.to_proto() for f in self._config.default_filters], input_json_schema=json_schema, + idempotency=( + self._config.idempotency.to_proto() + if self._config.idempotency + else None + ), ) def _get_workflow_input(self, ctx: Context) -> TWorkflowInput: @@ -399,7 +404,7 @@ def create_bulk_run_item( """ return WorkflowRunTriggerConfig( workflow_name=self._config.name, - input=self._serialize_input(input, target="string"), + input=self._serialize_input(input, target="bytes"), options=self._create_trigger_run_options_with_combined_additional_meta( options, child_key=child_key, @@ -412,7 +417,7 @@ def create_bulk_run_item( key=key, ) - def _serialize_input_to_str(self, input: TWorkflowInput | None) -> str | None: + def _serialize_input_to_bytes(self, input: TWorkflowInput | None) -> str | None: return self._config.input_validator.dump_json( input, # type: ignore[arg-type] context=HATCHET_PYDANTIC_SENTINEL, @@ -432,7 +437,7 @@ def _serialize_input_to_dict( @overload def _serialize_input( - self, input: TWorkflowInput | None, target: Literal["string"] = "string" + self, input: TWorkflowInput | None, target: Literal["bytes"] = "bytes" ) -> str | None: ... @overload @@ -443,13 +448,13 @@ def _serialize_input( def _serialize_input( self, input: TWorkflowInput | None, - target: Literal["string"] | Literal["dict"] = "string", + target: Literal["bytes"] | Literal["dict"] = "bytes", ) -> JSONSerializableMapping | str | None: if not input: return None - if target == "string": - return self._serialize_input_to_str(input) + if target == "bytes": + return self._serialize_input_to_bytes(input) if target == "dict": return self._serialize_input_to_dict(input) @@ -636,7 +641,7 @@ def schedule( return self._client._client.admin.schedule_workflow( name=self._config.name, schedules=[run_at], - input=self._serialize_input(input, target="string"), + input=self._serialize_input(input, target="bytes"), options=opts, ) @@ -963,7 +968,7 @@ def run( ref = self._client._client.admin.run_workflow( workflow_name=self._config.name, - input=self._serialize_input(input, target="string"), + input=self._serialize_input(input, target="bytes"), options=self._create_trigger_run_options_with_combined_additional_meta( options, child_key=child_key, @@ -1101,7 +1106,7 @@ async def aio_run( if durable_ctx is not None and durable_ctx._supports_durable_eviction: config = WorkflowRunTriggerConfig( workflow_name=self._config.name, - input=self._serialize_input(input, target="string"), + input=self._serialize_input(input, target="bytes"), options=opts, ) durable_spawn_results = await durable_ctx._spawn_children_no_wait([config]) @@ -1125,7 +1130,7 @@ async def aio_run( ref = await self._client._client.admin.aio_run_workflow( workflow_name=self._config.name, - input=self._serialize_input(input, target="string"), + input=self._serialize_input(input, target="bytes"), options=opts, ) diff --git a/sdks/python/hatchet_sdk/types/idempotency.py b/sdks/python/hatchet_sdk/types/idempotency.py new file mode 100644 index 0000000000..2212120d8f --- /dev/null +++ b/sdks/python/hatchet_sdk/types/idempotency.py @@ -0,0 +1,18 @@ +from datetime import timedelta + +from pydantic import BaseModel + +from hatchet_sdk.contracts.v1.workflows_pb2 import ( + IdempotencyConfig as IdempotencyConfigProto, +) + + +class IdempotencyConfig(BaseModel): + key_expression: str + ttl: timedelta + + def to_proto(self) -> IdempotencyConfigProto: + return IdempotencyConfigProto( + expression=self.key_expression, + ttl_ms=int(self.ttl.total_seconds() * 1000), + ) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 5583c23b31..9ab44c57c3 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hatchet-sdk" -version = "1.33.17" +version = "1.34.0" description = "This is the official Python SDK for Hatchet, a distributed, fault-tolerant task queue. The SDK allows you to easily integrate Hatchet's task scheduling and workflow orchestration capabilities into your Python applications." readme = "README.md" license = { text = "MIT" } @@ -16,6 +16,7 @@ authors = [ dependencies = [ "grpcio>=1.76.0,<2", "grpcio-tools>=1.76.0,<2", + "grpcio-status>=1.76.0,<2", "protobuf>=6.30.2,<7", "pydantic>=2.6.3,<3", "python-dateutil>=2.9.0.post0,<3", @@ -47,6 +48,7 @@ lint = [ "types-requests>=2.33.0.20260518", "psutil>=6.0.0", "types-grpcio>=1.0.0.20260614", + "types-grpcio-status (>=1.0.0.20260408,<2.0.0.0)", ] test = [ diff --git a/sdks/ruby/examples/idempotency/test_idempotency_spec.rb b/sdks/ruby/examples/idempotency/test_idempotency_spec.rb new file mode 100644 index 0000000000..32a1ed6d99 --- /dev/null +++ b/sdks/ruby/examples/idempotency/test_idempotency_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require "securerandom" +require_relative "../spec_helper" +require_relative "../worker_fixture" +require_relative "worker" + +RSpec.describe "Idempotency" do + around do |example| + HatchetWorkerFixture.with_worker( + ["bundle", "exec", "ruby", File.expand_path("worker.rb", __dir__)], + healthcheck_port: 8005 + ) do + example.run + end + end + + def wait_for_runs(test_run_id, min_runs:) + runs = nil + + 15.times do + runs = hatchet.runs.list( + since: Time.now - (5 * 60), + additional_metadata: { "test_run_id" => test_run_id }, + limit: 20 + ) + + break if runs.rows.length >= min_runs + + sleep 1 + end + + runs + end + + it "prevents duplicate direct triggers" do + test_run_id = SecureRandom.uuid + ref1 = IDEMPOTENT_TASK.run_no_wait( + { "id" => test_run_id }, + options: Hatchet::TriggerWorkflowOptions.new( + additional_metadata: { "test_run_id" => test_run_id } + ) + ) + + collision = nil + + expect do + IDEMPOTENT_TASK.run_no_wait({ "id" => test_run_id }) + end.to raise_error(Hatchet::IdempotencyCollisionError) { |error| collision = error } + + expect(collision&.existing_run_external_id).to eq(ref1.workflow_run_id) + + runs = wait_for_runs(test_run_id, min_runs: 1) + + expect(runs).not_to be_nil + expect(runs.rows.length).to eq(1) + expect(runs.rows[0].metadata.id).to eq(ref1.workflow_run_id) + end + + it "allows reruns after the short idempotency window expires" do + test_run_id = SecureRandom.uuid + + 4.times do |i| + if i == 1 + expect do + IDEMPOTENT_TASK_SHORT_WINDOW.run_no_wait( + { "id" => test_run_id }, + options: Hatchet::TriggerWorkflowOptions.new( + additional_metadata: { "test_run_id" => test_run_id } + ) + ) + end.to raise_error(Hatchet::IdempotencyCollisionError) { |error| expect(error.existing_run_external_id).not_to be_nil } + else + IDEMPOTENT_TASK_SHORT_WINDOW.run_no_wait( + { "id" => test_run_id }, + options: Hatchet::TriggerWorkflowOptions.new( + additional_metadata: { "test_run_id" => test_run_id } + ) + ) + end + + sleep(i + 1.5) unless i == 3 + end + + runs = wait_for_runs(test_run_id, min_runs: 3) + + expect(runs.rows.length).to eq(3) + end + + it "deduplicates event-triggered runs" do + test_run_id = SecureRandom.uuid + e1 = hatchet.events.push( + EVENT_KEY, + { "id" => test_run_id }, + additional_metadata: { "test_run_id" => test_run_id } + ) + e2 = hatchet.events.push( + EVENT_KEY, + { "id" => test_run_id }, + additional_metadata: { "test_run_id" => test_run_id } + ) + + runs = wait_for_runs(test_run_id, min_runs: 1) + + expect(runs).not_to be_nil + expect(runs.rows.length).to eq(1) + + details = nil + 15.times do + details = hatchet.events.list(event_ids: [e1.event_id, e2.event_id]) + break if details.rows.length == 2 + + sleep 1 + end + + expect(details).not_to be_nil + expect(details.rows.length).to eq(2) + + all_triggered_runs = details.rows.flat_map { |row| row.triggered_runs || [] } + + expect(all_triggered_runs.length).to eq(1) + + run = nil + 15.times do + run = hatchet.runs.get(all_triggered_runs[0].workflow_run_id) + break unless %w[QUEUED RUNNING].include?(run.status) + + sleep 1 + end + + expect(run.status).to eq("COMPLETED") + end +end diff --git a/sdks/ruby/examples/idempotency/trigger.rb b/sdks/ruby/examples/idempotency/trigger.rb new file mode 100644 index 0000000000..486ed35155 --- /dev/null +++ b/sdks/ruby/examples/idempotency/trigger.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'hatchet-sdk' +require_relative 'worker' + +HATCHET = Hatchet::Client.new(debug: true) unless defined?(HATCHET) + +# > trigger +first_ref = IDEMPOTENT_TASK.run_no_wait({ 'id' => '123' }) + +second_run_id = begin + second_ref = IDEMPOTENT_TASK.run_no_wait({ 'id' => '123' }) + second_ref.workflow_run_id +rescue Hatchet::IdempotencyCollisionError => e + puts "Run #{e.existing_run_external_id} already exists for this idempotency key" + e.existing_run_external_id +end +# !! + +puts "First run: #{first_ref.workflow_run_id}" +puts "Second run (or existing): #{second_run_id}" diff --git a/sdks/ruby/examples/idempotency/worker.rb b/sdks/ruby/examples/idempotency/worker.rb new file mode 100644 index 0000000000..6c07372b9d --- /dev/null +++ b/sdks/ruby/examples/idempotency/worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'hatchet-sdk' + +HATCHET = Hatchet::Client.new(debug: true) unless defined?(HATCHET) +EVENT_KEY = 'ruby-e2e:idempotency-example' + +# > idempotency +IDEMPOTENT_TASK = HATCHET.task( + name: 'ruby-e2e-idempotent-task', + idempotency: { expression: 'input.id', ttl_ms: 60_000 }, + on_events: [EVENT_KEY] +) do |input, _ctx| + { 'result' => "Hello from task #{input['id']}" } +end + +IDEMPOTENT_TASK_SHORT_WINDOW = HATCHET.task( + name: 'ruby-e2e-idempotent-task-short-window', + idempotency: { expression: 'input.id', ttl_ms: 2_000 } +) do |input, _ctx| + { 'result' => "Hello from task #{input['id']}" } +end +# !! + +def main + worker = HATCHET.worker('idempotency-worker', workflows: [IDEMPOTENT_TASK, IDEMPOTENT_TASK_SHORT_WINDOW]) + worker.start +end + +main if __FILE__ == $PROGRAM_NAME diff --git a/sdks/ruby/src/.rubocop.yml b/sdks/ruby/src/.rubocop.yml index 30d92223e8..0b58d10910 100644 --- a/sdks/ruby/src/.rubocop.yml +++ b/sdks/ruby/src/.rubocop.yml @@ -28,9 +28,6 @@ Style/FrozenStringLiteralComment: Style/Documentation: Enabled: false -# TODO(gregfurman): Requires refactoring classes in lib/hatchet-sdk.rb:73:1 -Style/OneClassPerFile: - Enabled: false Style/TrailingCommaInArguments: EnforcedStyleForMultiline: consistent_comma diff --git a/sdks/ruby/src/CHANGELOG.md b/sdks/ruby/src/CHANGELOG.md index 34a2fba4f7..8995bb6fda 100644 --- a/sdks/ruby/src/CHANGELOG.md +++ b/sdks/ruby/src/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Hatchet's Ruby SDK will be documented in this changelog. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2026-06-03 + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks via an `idempotency` option, which ensures that they're only run once in a provided time window, based on a CEL expression. Triggers that collide with an existing run raise an `IdempotencyCollisionError` containing the existing run's ID. + ## [0.3.1] - 2026-06-12 ### Fixed diff --git a/sdks/ruby/src/Gemfile.lock b/sdks/ruby/src/Gemfile.lock index 2f95462b4a..444def408d 100644 --- a/sdks/ruby/src/Gemfile.lock +++ b/sdks/ruby/src/Gemfile.lock @@ -1,11 +1,12 @@ PATH remote: . specs: - hatchet-sdk (0.3.1) + hatchet-sdk (0.4.0) concurrent-ruby (>= 1.1) faraday (~> 2.0) faraday-multipart google-protobuf (~> 4.0) + googleapis-common-protos-types (~> 1.0) grpc (~> 1.60) json (~> 2.0) marcel diff --git a/sdks/ruby/src/hatchet-sdk.gemspec b/sdks/ruby/src/hatchet-sdk.gemspec index 09692d52e2..f45d02dfe4 100644 --- a/sdks/ruby/src/hatchet-sdk.gemspec +++ b/sdks/ruby/src/hatchet-sdk.gemspec @@ -57,6 +57,7 @@ Gem::Specification.new do |spec| # Runtime dependencies for gRPC spec.add_dependency "concurrent-ruby", ">= 1.1" + spec.add_dependency "googleapis-common-protos-types", "~> 1.0" spec.add_dependency "google-protobuf", "~> 4.0" spec.add_dependency "grpc", "~> 1.60" diff --git a/sdks/ruby/src/lib/hatchet-sdk.rb b/sdks/ruby/src/lib/hatchet-sdk.rb index 3bd264c21a..2b91f4b3a8 100644 --- a/sdks/ruby/src/lib/hatchet-sdk.rb +++ b/sdks/ruby/src/lib/hatchet-sdk.rb @@ -3,80 +3,71 @@ require_relative "hatchet/version" require_relative "hatchet/config" -# Define base error class before loading submodules that depend on it -module Hatchet - # Base error class for all Hatchet-related errors - class Error < StandardError; end -end - -require_relative "hatchet/clients" -require_relative "hatchet/features/events" -require_relative "hatchet/features/runs" -require_relative "hatchet/features/tenant" -require_relative "hatchet/features/logs" -require_relative "hatchet/features/workers" -require_relative "hatchet/features/cel" -require_relative "hatchet/features/workflows" -require_relative "hatchet/features/filters" -require_relative "hatchet/features/metrics" -require_relative "hatchet/features/rate_limits" -require_relative "hatchet/features/cron" -require_relative "hatchet/features/scheduled" - -# Core classes -require_relative "hatchet/exceptions" -require_relative "hatchet/engine_version" -require_relative "hatchet/eviction_policy" -require_relative "hatchet/concurrency" -require_relative "hatchet/conditions" -require_relative "hatchet/condition_converter" -require_relative "hatchet/rate_limit" -require_relative "hatchet/labels" -require_relative "hatchet/trigger_options" -require_relative "hatchet/default_filter" -require_relative "hatchet/workflow_run" -require_relative "hatchet/context" -require_relative "hatchet/durable_context" -require_relative "hatchet/task" -require_relative "hatchet/workflow" -require_relative "hatchet/context_vars" -require_relative "hatchet/worker_obj" - -# gRPC connection and client infrastructure -require_relative "hatchet/connection" - -# Generated protobuf contracts (add contracts directory to load path for internal requires) -$LOAD_PATH.unshift(File.join(__dir__, "hatchet", "contracts")) unless $LOAD_PATH.include?(File.join(__dir__, "hatchet", "contracts")) -require_relative "hatchet/contracts/dispatcher/dispatcher_pb" -require_relative "hatchet/contracts/dispatcher/dispatcher_services_pb" -require_relative "hatchet/contracts/events/events_pb" -require_relative "hatchet/contracts/events/events_services_pb" -require_relative "hatchet/contracts/workflows/workflows_pb" -require_relative "hatchet/contracts/workflows/workflows_services_pb" -require_relative "hatchet/contracts/v1/shared/condition_pb" -require_relative "hatchet/contracts/v1/shared/trigger_pb" -require_relative "hatchet/contracts/v1/dispatcher_pb" -require_relative "hatchet/contracts/v1/dispatcher_services_pb" -require_relative "hatchet/contracts/v1/workflows_pb" -require_relative "hatchet/contracts/v1/workflows_services_pb" - -# gRPC client wrappers -require_relative "hatchet/clients/grpc/dispatcher" -require_relative "hatchet/clients/grpc/admin" -require_relative "hatchet/clients/grpc/event_client" - -# Worker runtime -require_relative "hatchet/worker/action_listener" -require_relative "hatchet/worker/workflow_run_listener" -require_relative "hatchet/worker/durable_eviction/cache" -require_relative "hatchet/worker/durable_eviction/manager" -require_relative "hatchet/worker/durable_event_listener" -require_relative "hatchet/worker/runner" - # Ruby SDK for Hatchet workflow engine # # @see https://docs.hatchet.run for Hatchet documentation module Hatchet + class Error < StandardError; end + + require_relative "hatchet/clients" + require_relative "hatchet/features/events" + require_relative "hatchet/features/runs" + require_relative "hatchet/features/tenant" + require_relative "hatchet/features/logs" + require_relative "hatchet/features/workers" + require_relative "hatchet/features/cel" + require_relative "hatchet/features/workflows" + require_relative "hatchet/features/filters" + require_relative "hatchet/features/metrics" + require_relative "hatchet/features/rate_limits" + require_relative "hatchet/features/cron" + require_relative "hatchet/features/scheduled" + + require_relative "hatchet/exceptions" + require_relative "hatchet/engine_version" + require_relative "hatchet/eviction_policy" + require_relative "hatchet/concurrency" + require_relative "hatchet/conditions" + require_relative "hatchet/condition_converter" + require_relative "hatchet/rate_limit" + require_relative "hatchet/labels" + require_relative "hatchet/trigger_options" + require_relative "hatchet/default_filter" + require_relative "hatchet/workflow_run" + require_relative "hatchet/context" + require_relative "hatchet/durable_context" + require_relative "hatchet/task" + require_relative "hatchet/workflow" + require_relative "hatchet/context_vars" + require_relative "hatchet/worker_obj" + + require_relative "hatchet/connection" + + $LOAD_PATH.unshift(File.join(__dir__, "hatchet", "contracts")) unless $LOAD_PATH.include?(File.join(__dir__, "hatchet", "contracts")) + require_relative "hatchet/contracts/dispatcher/dispatcher_pb" + require_relative "hatchet/contracts/dispatcher/dispatcher_services_pb" + require_relative "hatchet/contracts/events/events_pb" + require_relative "hatchet/contracts/events/events_services_pb" + require_relative "hatchet/contracts/workflows/workflows_pb" + require_relative "hatchet/contracts/workflows/workflows_services_pb" + require_relative "hatchet/contracts/v1/shared/condition_pb" + require_relative "hatchet/contracts/v1/shared/trigger_pb" + require_relative "hatchet/contracts/v1/dispatcher_pb" + require_relative "hatchet/contracts/v1/dispatcher_services_pb" + require_relative "hatchet/contracts/v1/workflows_pb" + require_relative "hatchet/contracts/v1/workflows_services_pb" + + require_relative "hatchet/clients/grpc/dispatcher" + require_relative "hatchet/clients/grpc/admin" + require_relative "hatchet/clients/grpc/event_client" + + require_relative "hatchet/worker/action_listener" + require_relative "hatchet/worker/workflow_run_listener" + require_relative "hatchet/worker/durable_eviction/cache" + require_relative "hatchet/worker/durable_eviction/manager" + require_relative "hatchet/worker/durable_event_listener" + require_relative "hatchet/worker/runner" + # The main client for interacting with Hatchet services. # # @example Basic usage with API token @@ -195,7 +186,7 @@ def scheduled # Create a new workflow definition # # @param name [String] Workflow name - # @param opts [Hash] Workflow options + # @param opts [Hash] Workflow options (on_events:, concurrency:, idempotency:, etc.) # @return [Hatchet::Workflow] # # @example @@ -208,17 +199,17 @@ def workflow(name:, **opts) # Create a standalone task (auto-wraps in a single-task workflow) # # @param name [String] Task name - # @param opts [Hash] Task options + # @param opts [Hash] Task options (on_events:, idempotency:, retries:, etc.) # @yield [input, ctx] The task execution block # @return [Hatchet::Task] # # @example # my_task = hatchet.task(name: "my_task") { |input, ctx| { "result" => "done" } } def task(name:, **opts, &block) - # Create a workflow wrapper for standalone tasks wf = Workflow.new(name: name, client: self, on_events: opts.delete(:on_events) || [], - default_filters: opts.delete(:default_filters) || [],) + default_filters: opts.delete(:default_filters) || [], + idempotency: opts.delete(:idempotency),) wf.task(name, **opts, &block) end diff --git a/sdks/ruby/src/lib/hatchet/clients/grpc/admin.rb b/sdks/ruby/src/lib/hatchet/clients/grpc/admin.rb index 4c78f120cc..daead5c810 100644 --- a/sdks/ruby/src/lib/hatchet/clients/grpc/admin.rb +++ b/sdks/ruby/src/lib/hatchet/clients/grpc/admin.rb @@ -2,6 +2,7 @@ require "json" require "google/protobuf/timestamp_pb" +require "google/rpc/status_pb" module Hatchet module Clients @@ -93,6 +94,9 @@ def trigger_workflow(workflow_name, input: {}, options: {}) response = @v0_stub.trigger_workflow(request, metadata: @config.auth_metadata) response.workflow_run_id rescue ::GRPC::AlreadyExists => e + run_id = extract_idempotency_run_id(e) + raise(IdempotencyCollisionError, run_id) if run_id + raise DedupeViolationError, "Deduplication violation: #{e.message}" rescue ::GRPC::ResourceExhausted => e raise ResourceExhaustedError, e.message @@ -284,6 +288,21 @@ def build_trigger_worker_labels(labels) end end + def extract_idempotency_run_id(grpc_error) + status_bin = grpc_error.to_status.metadata&.[]("grpc-status-details-bin") + return nil unless status_bin + + rpc_status = Google::Rpc::Status.decode(status_bin.b) + rpc_status.details.each do |any| + next unless any.type_url.include?("IdempotencyCollisionError") + + return ::V1::IdempotencyCollisionError.decode(any.value).existing_run_external_id + end + nil + rescue StandardError + nil + end + def ensure_connected! return if @v0_stub && @v1_stub diff --git a/sdks/ruby/src/lib/hatchet/contracts/v1/workflows_pb.rb b/sdks/ruby/src/lib/hatchet/contracts/v1/workflows_pb.rb index 628b311eea..af59513de3 100644 --- a/sdks/ruby/src/lib/hatchet/contracts/v1/workflows_pb.rb +++ b/sdks/ruby/src/lib/hatchet/contracts/v1/workflows_pb.rb @@ -9,7 +9,7 @@ require 'v1/shared/trigger_pb' -descriptor_data = "\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xae\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priority\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xac\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schema\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3" +descriptor_data = "\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xae\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priority\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xed\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x12/\n\x0bidempotency\x18\x0f \x01(\x0b\x32\x15.v1.IdempotencyConfigH\x05\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schemaB\x0e\n\x0c_idempotency\"7\n\x11IdempotencyConfig\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x0e\n\x06ttl_ms\x18\x02 \x01(\x03\"=\n\x19IdempotencyCollisionError\x12 \n\x18\x65xisting_run_external_id\x18\x01 \x01(\t\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3" pool = ::Google::Protobuf::DescriptorPool.generated_pool pool.add_serialized_file(descriptor_data) @@ -25,6 +25,8 @@ module V1 BranchDurableTaskRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.BranchDurableTaskRequest").msgclass BranchDurableTaskResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.BranchDurableTaskResponse").msgclass CreateWorkflowVersionRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.CreateWorkflowVersionRequest").msgclass + IdempotencyConfig = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.IdempotencyConfig").msgclass + IdempotencyCollisionError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.IdempotencyCollisionError").msgclass DefaultFilter = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.DefaultFilter").msgclass Concurrency = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.Concurrency").msgclass CreateTaskOpts = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("v1.CreateTaskOpts").msgclass diff --git a/sdks/ruby/src/lib/hatchet/exceptions.rb b/sdks/ruby/src/lib/hatchet/exceptions.rb index 0f81e47a29..23ec21ed7a 100644 --- a/sdks/ruby/src/lib/hatchet/exceptions.rb +++ b/sdks/ruby/src/lib/hatchet/exceptions.rb @@ -24,6 +24,18 @@ def initialize(message = "Dedupe violation: a run with this key already exists") end end + # Raised when an idempotency key collision occurs. + # Contains the ID of the existing run that claimed the key. + class IdempotencyCollisionError < Error + # @return [String] The external ID of the existing workflow run + attr_reader :existing_run_external_id + + def initialize(existing_run_external_id) + @existing_run_external_id = existing_run_external_id + super("idempotency key collision: existing run #{existing_run_external_id} already exists") + end + end + # Represents an error from a failed task run class TaskRunError < Error # @return [String] The external ID of the failed task run diff --git a/sdks/ruby/src/lib/hatchet/version.rb b/sdks/ruby/src/lib/hatchet/version.rb index fc19ccf4cb..c09c6fb6e2 100644 --- a/sdks/ruby/src/lib/hatchet/version.rb +++ b/sdks/ruby/src/lib/hatchet/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Hatchet - VERSION = "0.3.1" + VERSION = "0.4.0" end diff --git a/sdks/ruby/src/lib/hatchet/workflow.rb b/sdks/ruby/src/lib/hatchet/workflow.rb index 5f242d0220..589220d696 100644 --- a/sdks/ruby/src/lib/hatchet/workflow.rb +++ b/sdks/ruby/src/lib/hatchet/workflow.rb @@ -46,6 +46,9 @@ class Workflow # @return [Task, nil] The on_success task attr_reader :on_success + # @return [Hash, nil] Idempotency config with :expression and :ttl_ms keys + attr_reader :idempotency + # @return [String, nil] The workflow ID writer (set after registration) attr_writer :id @@ -57,6 +60,7 @@ class Workflow # @param task_defaults [Hash, nil] Default task settings # @param default_filters [Array] Default filters # @param sticky [Symbol, nil] Sticky strategy + # @param idempotency [Hash, nil] Idempotency config with :expression and :ttl_ms keys # @param client [Hatchet::Client, nil] The client def initialize( name:, @@ -67,6 +71,7 @@ def initialize( task_defaults: nil, default_filters: [], sticky: nil, + idempotency: nil, client: nil ) @name = name @@ -78,6 +83,7 @@ def initialize( @task_defaults = task_defaults @default_filters = default_filters @sticky = sticky + @idempotency = idempotency @client = client @on_failure = nil @on_success = nil @@ -208,6 +214,13 @@ def to_proto(config) args[:default_priority] = @default_priority if @default_priority args[:default_filters] = filter_protos unless filter_protos.empty? + if @idempotency + args[:idempotency] = ::V1::IdempotencyConfig.new( + expression: @idempotency[:expression], + ttl_ms: @idempotency[:ttl_ms], + ) + end + ::V1::CreateWorkflowVersionRequest.new(**args) end diff --git a/sdks/typescript/CHANGELOG.md b/sdks/typescript/CHANGELOG.md index 55d479d983..abc1072c94 100644 --- a/sdks/typescript/CHANGELOG.md +++ b/sdks/typescript/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Hatchet's TypeScript SDK will be documented in this chang The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.24.0] - 2026-06-03 + +### Added + +- Adds support for defining **idempotency keys** on workflows and standalone tasks via an `idempotency` option, which ensures that they're only run once in a provided time window, based on a CEL expression. Triggers that collide with an existing run throw an `IdempotencyCollisionError` containing the existing run's ID. + ## [1.24.3] - 2026-06-17 ### Removed diff --git a/sdks/typescript/generate-protoc.sh b/sdks/typescript/generate-protoc.sh index 39c8ea155f..05243db431 100755 --- a/sdks/typescript/generate-protoc.sh +++ b/sdks/typescript/generate-protoc.sh @@ -4,8 +4,10 @@ OUT_DIR="./src/protoc" if [ -d "./hatchet" ]; then IN_DIR="./hatchet/api-contracts" + VENDOR_DIR="./hatchet/hack/proto/vendor" else IN_DIR="../../api-contracts" + VENDOR_DIR="../../hack/proto/vendor" fi # Generate code @@ -14,6 +16,8 @@ fi --ts_proto_out=$OUT_DIR \ --ts_proto_opt=outputServices=nice-grpc,outputServices=generic-definitions,useExactTypes=false \ --proto_path=$IN_DIR \ - $IN_DIR/**/*.proto + --proto_path=$VENDOR_DIR \ + $IN_DIR/**/*.proto \ + $VENDOR_DIR/google/rpc/status.proto pnpm lint:fix diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index e970dd0944..e95d22a3d4 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@hatchet-dev/typescript-sdk", - "version": "1.24.3", + "version": "1.24.0", "description": "Background task orchestration & visibility for developers", "types": "dist/index.d.ts", "files": [ diff --git a/sdks/typescript/src/protoc/google/protobuf/any.ts b/sdks/typescript/src/protoc/google/protobuf/any.ts new file mode 100644 index 0000000000..3bbab777c9 --- /dev/null +++ b/sdks/typescript/src/protoc/google/protobuf/any.ts @@ -0,0 +1,258 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v2.11.6 +// protoc v3.19.1 +// source: google/protobuf/any.proto + +/* eslint-disable */ +import { BinaryReader, BinaryWriter } from '@bufbuild/protobuf/wire'; + +export const protobufPackage = 'google.protobuf'; + +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + */ +export interface Any { + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + */ + typeUrl: string; + /** Must be a valid serialized protocol buffer of the above specified type. */ + value: Uint8Array; +} + +function createBaseAny(): Any { + return { typeUrl: '', value: new Uint8Array(0) }; +} + +export const Any: MessageFns = { + encode(message: Any, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.typeUrl !== '') { + writer.uint32(10).string(message.typeUrl); + } + if (message.value.length !== 0) { + writer.uint32(18).bytes(message.value); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): Any { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAny(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.typeUrl = reader.string(); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.value = reader.bytes(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): Any { + return { + typeUrl: isSet(object.typeUrl) + ? globalThis.String(object.typeUrl) + : isSet(object.type_url) + ? globalThis.String(object.type_url) + : '', + value: isSet(object.value) ? bytesFromBase64(object.value) : new Uint8Array(0), + }; + }, + + toJSON(message: Any): unknown { + const obj: any = {}; + if (message.typeUrl !== '') { + obj.typeUrl = message.typeUrl; + } + if (message.value.length !== 0) { + obj.value = base64FromBytes(message.value); + } + return obj; + }, + + create(base?: DeepPartial): Any { + return Any.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): Any { + const message = createBaseAny(); + message.typeUrl = object.typeUrl ?? ''; + message.value = object.value ?? new Uint8Array(0); + return message; + }, +}; + +function bytesFromBase64(b64: string): Uint8Array { + if ((globalThis as any).Buffer) { + return Uint8Array.from((globalThis as any).Buffer.from(b64, 'base64')); + } else { + const bin = globalThis.atob(b64); + const arr = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; ++i) { + arr[i] = bin.charCodeAt(i); + } + return arr; + } +} + +function base64FromBytes(arr: Uint8Array): string { + if ((globalThis as any).Buffer) { + return (globalThis as any).Buffer.from(arr).toString('base64'); + } else { + const bin: string[] = []; + arr.forEach((byte) => { + bin.push(globalThis.String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join('')); + } +} + +type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; + +export type DeepPartial = T extends Builtin + ? T + : T extends globalThis.Array + ? globalThis.Array> + : T extends ReadonlyArray + ? ReadonlyArray> + : T extends {} + ? { [K in keyof T]?: DeepPartial } + : Partial; + +function isSet(value: any): boolean { + return value !== null && value !== undefined; +} + +export interface MessageFns { + encode(message: T, writer?: BinaryWriter): BinaryWriter; + decode(input: BinaryReader | Uint8Array, length?: number): T; + fromJSON(object: any): T; + toJSON(message: T): unknown; + create(base?: DeepPartial): T; + fromPartial(object: DeepPartial): T; +} diff --git a/sdks/typescript/src/protoc/google/rpc/status.ts b/sdks/typescript/src/protoc/google/rpc/status.ts new file mode 100644 index 0000000000..85f43dede6 --- /dev/null +++ b/sdks/typescript/src/protoc/google/rpc/status.ts @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v2.11.6 +// protoc v3.19.1 +// source: google/rpc/status.proto + +/* eslint-disable */ +import { BinaryReader, BinaryWriter } from '@bufbuild/protobuf/wire'; +import { Any } from '../protobuf/any'; + +export const protobufPackage = 'google.rpc'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + */ +export interface Status { + /** + * The status code, which should be an enum value of + * [google.rpc.Code][google.rpc.Code]. + */ + code: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized + * by the client. + */ + message: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + */ + details: Any[]; +} + +function createBaseStatus(): Status { + return { code: 0, message: '', details: [] }; +} + +export const Status: MessageFns = { + encode(message: Status, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.code !== 0) { + writer.uint32(8).int32(message.code); + } + if (message.message !== '') { + writer.uint32(18).string(message.message); + } + for (const v of message.details) { + Any.encode(v!, writer.uint32(26).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): Status { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStatus(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 8) { + break; + } + + message.code = reader.int32(); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.message = reader.string(); + continue; + } + case 3: { + if (tag !== 26) { + break; + } + + message.details.push(Any.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): Status { + return { + code: isSet(object.code) ? globalThis.Number(object.code) : 0, + message: isSet(object.message) ? globalThis.String(object.message) : '', + details: globalThis.Array.isArray(object?.details) + ? object.details.map((e: any) => Any.fromJSON(e)) + : [], + }; + }, + + toJSON(message: Status): unknown { + const obj: any = {}; + if (message.code !== 0) { + obj.code = Math.round(message.code); + } + if (message.message !== '') { + obj.message = message.message; + } + if (message.details?.length) { + obj.details = message.details.map((e) => Any.toJSON(e)); + } + return obj; + }, + + create(base?: DeepPartial): Status { + return Status.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): Status { + const message = createBaseStatus(); + message.code = object.code ?? 0; + message.message = object.message ?? ''; + message.details = object.details?.map((e) => Any.fromPartial(e)) || []; + return message; + }, +}; + +type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; + +export type DeepPartial = T extends Builtin + ? T + : T extends globalThis.Array + ? globalThis.Array> + : T extends ReadonlyArray + ? ReadonlyArray> + : T extends {} + ? { [K in keyof T]?: DeepPartial } + : Partial; + +function isSet(value: any): boolean { + return value !== null && value !== undefined; +} + +export interface MessageFns { + encode(message: T, writer?: BinaryWriter): BinaryWriter; + decode(input: BinaryReader | Uint8Array, length?: number): T; + fromJSON(object: any): T; + toJSON(message: T): unknown; + create(base?: DeepPartial): T; + fromPartial(object: DeepPartial): T; +} diff --git a/sdks/typescript/src/protoc/v1/workflows.ts b/sdks/typescript/src/protoc/v1/workflows.ts index 6e729c1514..dad601b25c 100644 --- a/sdks/typescript/src/protoc/v1/workflows.ts +++ b/sdks/typescript/src/protoc/v1/workflows.ts @@ -312,6 +312,20 @@ export interface CreateWorkflowVersionRequest { defaultFilters: DefaultFilter[]; /** (optional) the JSON schema for the workflow input */ inputJsonSchema?: Uint8Array | undefined; + /** (optional) idempotency configuration for the workflow */ + idempotency?: IdempotencyConfig | undefined; +} + +export interface IdempotencyConfig { + /** a CEL expression for determining the idempotency key for workflow runs */ + expression: string; + /** time-to-live for idempotency keys in milliseconds */ + ttlMs: number; +} + +export interface IdempotencyCollisionError { + /** the external ID of the existing workflow run that caused the collision */ + existingRunExternalId: string; } export interface DefaultFilter { @@ -1448,6 +1462,7 @@ function createBaseCreateWorkflowVersionRequest(): CreateWorkflowVersionRequest concurrencyArr: [], defaultFilters: [], inputJsonSchema: undefined, + idempotency: undefined, }; } @@ -1498,6 +1513,9 @@ export const CreateWorkflowVersionRequest: MessageFns Concurrency.fromPartial(e)) || []; message.defaultFilters = object.defaultFilters?.map((e) => DefaultFilter.fromPartial(e)) || []; message.inputJsonSchema = object.inputJsonSchema ?? undefined; + message.idempotency = + object.idempotency !== undefined && object.idempotency !== null + ? IdempotencyConfig.fromPartial(object.idempotency) + : undefined; + return message; + }, +}; + +function createBaseIdempotencyConfig(): IdempotencyConfig { + return { expression: '', ttlMs: 0 }; +} + +export const IdempotencyConfig: MessageFns = { + encode(message: IdempotencyConfig, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.expression !== '') { + writer.uint32(10).string(message.expression); + } + if (message.ttlMs !== 0) { + writer.uint32(16).int64(message.ttlMs); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): IdempotencyConfig { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseIdempotencyConfig(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.expression = reader.string(); + continue; + } + case 2: { + if (tag !== 16) { + break; + } + + message.ttlMs = longToNumber(reader.int64()); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): IdempotencyConfig { + return { + expression: isSet(object.expression) ? globalThis.String(object.expression) : '', + ttlMs: isSet(object.ttlMs) + ? globalThis.Number(object.ttlMs) + : isSet(object.ttl_ms) + ? globalThis.Number(object.ttl_ms) + : 0, + }; + }, + + toJSON(message: IdempotencyConfig): unknown { + const obj: any = {}; + if (message.expression !== '') { + obj.expression = message.expression; + } + if (message.ttlMs !== 0) { + obj.ttlMs = Math.round(message.ttlMs); + } + return obj; + }, + + create(base?: DeepPartial): IdempotencyConfig { + return IdempotencyConfig.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): IdempotencyConfig { + const message = createBaseIdempotencyConfig(); + message.expression = object.expression ?? ''; + message.ttlMs = object.ttlMs ?? 0; + return message; + }, +}; + +function createBaseIdempotencyCollisionError(): IdempotencyCollisionError { + return { existingRunExternalId: '' }; +} + +export const IdempotencyCollisionError: MessageFns = { + encode( + message: IdempotencyCollisionError, + writer: BinaryWriter = new BinaryWriter() + ): BinaryWriter { + if (message.existingRunExternalId !== '') { + writer.uint32(10).string(message.existingRunExternalId); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): IdempotencyCollisionError { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseIdempotencyCollisionError(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.existingRunExternalId = reader.string(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): IdempotencyCollisionError { + return { + existingRunExternalId: isSet(object.existingRunExternalId) + ? globalThis.String(object.existingRunExternalId) + : isSet(object.existing_run_external_id) + ? globalThis.String(object.existing_run_external_id) + : '', + }; + }, + + toJSON(message: IdempotencyCollisionError): unknown { + const obj: any = {}; + if (message.existingRunExternalId !== '') { + obj.existingRunExternalId = message.existingRunExternalId; + } + return obj; + }, + + create(base?: DeepPartial): IdempotencyCollisionError { + return IdempotencyCollisionError.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): IdempotencyCollisionError { + const message = createBaseIdempotencyCollisionError(); + message.existingRunExternalId = object.existingRunExternalId ?? ''; return message; }, }; diff --git a/sdks/typescript/src/util/errors/idempotency-collision-error.ts b/sdks/typescript/src/util/errors/idempotency-collision-error.ts new file mode 100644 index 0000000000..f64d0a2b0c --- /dev/null +++ b/sdks/typescript/src/util/errors/idempotency-collision-error.ts @@ -0,0 +1,12 @@ +import HatchetError from './hatchet-error'; + +export class IdempotencyCollisionError extends HatchetError { + existingRunExternalId: string; + + constructor(existingRunExternalId: string) { + super(`idempotency key collision: existing run ${existingRunExternalId} already exists`); + this.name = 'IdempotencyCollisionError'; + this.existingRunExternalId = existingRunExternalId; + Object.setPrototypeOf(this, new.target.prototype); + } +} diff --git a/sdks/typescript/src/util/retrier.ts b/sdks/typescript/src/util/retrier.ts index 0164ee6bca..f2b9b49f81 100644 --- a/sdks/typescript/src/util/retrier.ts +++ b/sdks/typescript/src/util/retrier.ts @@ -9,7 +9,8 @@ export async function retrier( fn: () => Promise, logger: Logger, retries: number = DEFAULT_RETRY_COUNT, - interval: number = DEFAULT_RETRY_INTERVAL + interval: number = DEFAULT_RETRY_INTERVAL, + shouldRetry: (e: unknown) => boolean = () => true ) { let lastError: Error | undefined; @@ -17,6 +18,9 @@ export async function retrier( try { return await fn(); } catch (e: unknown) { + if (!shouldRetry(e)) { + throw e; + } lastError = e instanceof Error ? e : new Error(String(e)); logger.error(`Error: ${lastError.message}`); diff --git a/sdks/typescript/src/v1/client/admin.ts b/sdks/typescript/src/v1/client/admin.ts index 87540053cc..bf8342b674 100644 --- a/sdks/typescript/src/v1/client/admin.ts +++ b/sdks/typescript/src/v1/client/admin.ts @@ -1,6 +1,15 @@ import HatchetError from '@util/errors/hatchet-error'; +import { IdempotencyCollisionError } from '@util/errors/idempotency-collision-error'; import { ClientConfig } from '@clients/hatchet-client/client-config'; import WorkflowRunRef from '@hatchet/util/workflow-run-ref'; +import { ServiceError, status as GrpcStatus } from '@grpc/grpc-js'; +import { Status as RpcStatus } from '@hatchet/protoc/google/rpc/status'; +import { IdempotencyCollisionError as IdempotencyCollisionErrorProto } from '@hatchet/protoc/v1/workflows'; +import { + ClientError, + Metadata as NiceGrpcMetadata, + Status as NiceGrpcStatus, +} from 'nice-grpc-common'; import { Priority, RateLimitDuration, RunsClient, WorkerLabelComparator } from '@hatchet/v1'; import { createGrpcClient } from '@hatchet/util/grpc-helpers'; @@ -22,6 +31,48 @@ import { batch } from '@hatchet/util/batch'; import { applyNamespace } from '@hatchet/util/apply-namespace'; import { DesiredWorkerLabels } from '@hatchet-dev/typescript-sdk/protoc/v1/shared/trigger'; +function isGrpcServiceError(e: unknown): e is ServiceError { + return e instanceof Error && 'code' in e && 'metadata' in e; +} + +function extractExistingRunIdFromGrpcError(e: ServiceError): string { + try { + const [binData] = e.metadata.get('grpc-status-details-bin'); + if (!binData) return ''; + + const status = RpcStatus.decode(binData instanceof Buffer ? binData : Buffer.from(binData)); + for (const detail of status.details) { + if (detail.typeUrl.includes('IdempotencyCollisionError')) { + return IdempotencyCollisionErrorProto.decode(detail.value).existingRunExternalId ?? ''; + } + } + } catch { + // ignore decoding errors + } + return ''; +} + +function isNiceGrpcAlreadyExists(e: unknown): e is ClientError { + return e instanceof ClientError && e.code === NiceGrpcStatus.ALREADY_EXISTS; +} + +function extractRunIdFromNiceGrpcMetadata(metadata: NiceGrpcMetadata | undefined): string { + if (!metadata) return ''; + try { + const binData = metadata.get('grpc-status-details-bin'); + if (!binData) return ''; + const status = RpcStatus.decode(binData); + for (const detail of status.details) { + if (detail.typeUrl.includes('IdempotencyCollisionError')) { + return IdempotencyCollisionErrorProto.decode(detail.value).existingRunExternalId ?? ''; + } + } + } catch { + // ignore decoding errors + } + return ''; +} + type DesiredWorkerLabelOpt = { value: string | number; required?: boolean; @@ -135,6 +186,8 @@ export class AdminClient { _standaloneTaskName?: string | undefined; } ) { + let trailerMetadata: NiceGrpcMetadata | undefined; + try { const computedName = applyNamespace(workflowName, this.config.namespace).toLowerCase(); @@ -163,8 +216,16 @@ export class AdminClient { }; const resp = await retrier( - async () => this.workflowsGrpc.triggerWorkflow(request), - this.logger + async () => + this.workflowsGrpc.triggerWorkflow(request, { + onTrailer: (trailer) => { + trailerMetadata = trailer; + }, + }), + this.logger, + undefined, + undefined, + (e) => !isNiceGrpcAlreadyExists(e) ); const id = resp.workflowRunId; @@ -179,8 +240,14 @@ export class AdminClient { ); await ref.getWorkflowRunId(); return ref; - } catch (e: any) { - throw new HatchetError(e.message); + } catch (e: unknown) { + if (isGrpcServiceError(e) && e.code === GrpcStatus.ALREADY_EXISTS) { + throw new IdempotencyCollisionError(extractExistingRunIdFromGrpcError(e)); + } + if (isNiceGrpcAlreadyExists(e)) { + throw new IdempotencyCollisionError(extractRunIdFromNiceGrpcMetadata(trailerMetadata)); + } + throw new HatchetError(e instanceof Error ? e.message : String(e)); } } diff --git a/sdks/typescript/src/v1/client/worker/worker-internal.ts b/sdks/typescript/src/v1/client/worker/worker-internal.ts index ffa409fc51..61d5ad83b9 100644 --- a/sdks/typescript/src/v1/client/worker/worker-internal.ts +++ b/sdks/typescript/src/v1/client/worker/worker-internal.ts @@ -421,6 +421,12 @@ export class InternalWorker { expression: f.expression, payload: f.payload ? new TextEncoder().encode(JSON.stringify(f.payload)) : undefined, })) ?? [], + idempotency: workflow.idempotency + ? { + expression: workflow.idempotency.expression, + ttlMs: workflow.idempotency.ttlMs, + } + : undefined, }); this.registeredWorkflowPromises.push(registeredWorkflow); await registeredWorkflow; diff --git a/sdks/typescript/src/v1/declaration.ts b/sdks/typescript/src/v1/declaration.ts index 24d19d507e..5b75bfc5e7 100644 --- a/sdks/typescript/src/v1/declaration.ts +++ b/sdks/typescript/src/v1/declaration.ts @@ -25,6 +25,7 @@ import { Concurrency, DurableTaskFn, WorkerLabelComparator, + IdempotencyConfig, } from './task'; import { Duration } from './client/duration'; import { MetricsClient } from './client/features/metrics'; @@ -208,6 +209,12 @@ export type CreateBaseWorkflowOpts = { * can be used on the dashboard for autocomplete. */ inputValidator?: z.ZodType; + + /** + * (optional) idempotency configuration for the workflow. + * Prevents more than one run from occurring for a given key within the TTL window. + */ + idempotency?: IdempotencyConfig; }; export type CreateTaskWorkflowOpts< diff --git a/sdks/typescript/src/v1/examples/e2e-worker.ts b/sdks/typescript/src/v1/examples/e2e-worker.ts index 1cc78776ce..f7649329ab 100644 --- a/sdks/typescript/src/v1/examples/e2e-worker.ts +++ b/sdks/typescript/src/v1/examples/e2e-worker.ts @@ -47,6 +47,7 @@ import { durableSleep } from './durable_sleep/workflow'; import { createLoggingWorkflow } from './logger/workflow'; import { nonRetryableWorkflow } from './non_retryable/workflow'; import { failureWorkflow } from './on_failure/workflow'; +import { idempotentTask, idempotentTaskShortWindow } from './idempotency/workflow'; import { lower } from './on_event/workflow'; import { returnExceptionsTask } from './return_exceptions/workflow'; import { runDetailTestWorkflow } from './run_details/workflow'; @@ -112,6 +113,8 @@ const workflows = [ createLoggingWorkflow(hatchet), nonRetryableWorkflow, failureWorkflow, + idempotentTask, + idempotentTaskShortWindow, lower, returnExceptionsTask, runDetailTestWorkflow, diff --git a/sdks/typescript/src/v1/examples/idempotency/idempotency.e2e.ts b/sdks/typescript/src/v1/examples/idempotency/idempotency.e2e.ts new file mode 100644 index 0000000000..a74af10428 --- /dev/null +++ b/sdks/typescript/src/v1/examples/idempotency/idempotency.e2e.ts @@ -0,0 +1,161 @@ +import sleep from '@hatchet/util/sleep'; +import { randomUUID } from 'crypto'; +import { V1TaskStatus } from '@hatchet/clients/rest/generated/data-contracts'; +import { IdempotencyCollisionError } from '@hatchet/v1'; +import { makeE2EClient, poll, startWorker, stopWorker } from '../__e2e__/harness'; +import { EVENT_KEY, idempotentTask, idempotentTaskShortWindow } from './workflow'; + +describe('idempotency-e2e', () => { + const hatchet = makeE2EClient(); + let worker: Awaited> | undefined; + + beforeAll(async () => { + worker = await startWorker({ + client: hatchet, + name: 'idempotency-e2e-worker', + workflows: [idempotentTask, idempotentTaskShortWindow], + }); + }); + + afterAll(async () => { + await stopWorker(worker); + }); + + async function waitForRuns(testRunId: string, minRuns: number) { + return poll( + async () => + hatchet.runs.list({ + additionalMetadata: { test_run_id: testRunId }, + onlyTasks: false, + limit: 20, + }), + { + timeoutMs: 30_000, + intervalMs: 200, + label: `runs for ${testRunId}`, + shouldStop: (response) => (response.rows || []).length >= minRuns, + } + ); + } + + it('prevents duplicate direct triggers', async () => { + const testRunId = randomUUID(); + const ref1 = await idempotentTask.runNoWait( + { id: testRunId }, + { + additionalMetadata: { + test_run_id: testRunId, + }, + } + ); + + let collisionError: IdempotencyCollisionError | undefined; + + try { + await idempotentTask.runNoWait({ id: testRunId }); + } catch (e) { + if (e instanceof IdempotencyCollisionError) { + collisionError = e; + } else { + throw e; + } + } + + expect(collisionError).toBeInstanceOf(IdempotencyCollisionError); + expect(collisionError?.existingRunExternalId).toBe(await ref1.getWorkflowRunId()); + + const runs = await waitForRuns(testRunId, 1); + + expect(runs.rows).toHaveLength(1); + expect(runs.rows?.[0]?.metadata.id).toBe(await ref1.getWorkflowRunId()); + }, 60_000); + + it('allows reruns after the short idempotency window expires', async () => { + const testRunId = randomUUID(); + + for (let i = 0; i < 4; i += 1) { + if (i === 1) { + let collisionError: IdempotencyCollisionError | undefined; + try { + await idempotentTaskShortWindow.runNoWait( + { id: testRunId }, + { additionalMetadata: { test_run_id: testRunId } } + ); + } catch (e) { + if (e instanceof IdempotencyCollisionError) { + collisionError = e; + } else { + throw e; + } + } + expect(collisionError).toBeInstanceOf(IdempotencyCollisionError); + expect(collisionError?.existingRunExternalId).toBeTruthy(); + } else { + await idempotentTaskShortWindow.runNoWait( + { id: testRunId }, + { additionalMetadata: { test_run_id: testRunId } } + ); + } + + if (i !== 3) { + await sleep((i + 1.5) * 1000); + } + } + + const runs = await waitForRuns(testRunId, 3); + + expect(runs.rows).toHaveLength(3); + }, 60_000); + + it('deduplicates event-triggered runs', async () => { + const testRunId = randomUUID(); + const e1 = await hatchet.events.push( + EVENT_KEY, + { id: testRunId }, + { + additionalMetadata: { + test_run_id: testRunId, + }, + } + ); + const e2 = await hatchet.events.push( + EVENT_KEY, + { id: testRunId }, + { + additionalMetadata: { + test_run_id: testRunId, + }, + } + ); + + const runs = await waitForRuns(testRunId, 1); + + expect(runs.rows).toHaveLength(1); + + const details = await poll( + async () => hatchet.events.list({ eventIds: [e1.eventId, e2.eventId] }), + { + timeoutMs: 30_000, + intervalMs: 200, + label: 'idempotency event details', + shouldStop: (response) => (response.rows || []).length === 2, + } + ); + + const allTriggeredRuns = (details.rows || []).flatMap((row) => row.triggeredRuns || []); + + expect(allTriggeredRuns).toHaveLength(1); + + const runDetails = await poll(async () => hatchet.runs.get(allTriggeredRuns[0].workflowRunId), { + timeoutMs: 30_000, + intervalMs: 200, + label: 'idempotent event-triggered run completion', + shouldStop: (response) => { + const status = response.run?.status; + return status !== V1TaskStatus.QUEUED && status !== V1TaskStatus.RUNNING; + }, + }); + + expect(runDetails.run?.status).toBe(V1TaskStatus.COMPLETED); + }, 60_000); +}); diff --git a/sdks/typescript/src/v1/examples/idempotency/run.ts b/sdks/typescript/src/v1/examples/idempotency/run.ts new file mode 100644 index 0000000000..bc2372ac16 --- /dev/null +++ b/sdks/typescript/src/v1/examples/idempotency/run.ts @@ -0,0 +1,28 @@ +import { IdempotencyCollisionError } from '@hatchet/v1'; +import { idempotentTask } from './workflow'; + +async function main() { + // > trigger + const ref1 = await idempotentTask.runNoWait({ id: '123' }); + + let runId2: string; + try { + const ref2 = await idempotentTask.runNoWait({ id: '123' }); + runId2 = await ref2.getWorkflowRunId(); + } catch (e) { + if (e instanceof IdempotencyCollisionError) { + console.log( + `Run with external ID ${e.existingRunExternalId} already exists for this idempotency key` + ); + runId2 = e.existingRunExternalId; + } else { + throw e; + } + } + + const res1 = await ref1.result(); + console.log(`Result: ${JSON.stringify(res1)}, run ID: ${runId2}`); + // !! +} + +main().catch(console.error); diff --git a/sdks/typescript/src/v1/examples/idempotency/workflow.ts b/sdks/typescript/src/v1/examples/idempotency/workflow.ts new file mode 100644 index 0000000000..81b052abee --- /dev/null +++ b/sdks/typescript/src/v1/examples/idempotency/workflow.ts @@ -0,0 +1,32 @@ +import { hatchet } from '../hatchet-client'; + +export const EVENT_KEY = 'ts-e2e:idempotency-example'; + +export type IdempotencyInput = { + id: string; +}; + +// > idempotency +export const idempotentTask = hatchet.task({ + name: 'ts-e2e-idempotent-task', + idempotency: { + expression: 'input.id', + ttlMs: 60_000, + }, + onEvents: [EVENT_KEY], + fn: async (input) => { + return { result: `Hello, world from task ${input.id}` }; + }, +}); +// !! + +export const idempotentTaskShortWindow = hatchet.task({ + name: 'ts-e2e-idempotent-task-short-window', + idempotency: { + expression: 'input.id', + ttlMs: 2_000, + }, + fn: async (input) => { + return { result: `Hello, world from task ${input.id}` }; + }, +}); diff --git a/sdks/typescript/src/v1/index.ts b/sdks/typescript/src/v1/index.ts index c0dc0d0cea..cb601e1333 100644 --- a/sdks/typescript/src/v1/index.ts +++ b/sdks/typescript/src/v1/index.ts @@ -12,6 +12,7 @@ export * from './slot-types'; export * from '../legacy/legacy-transformer'; export { NonDeterminismError } from '../util/errors/non-determinism-error'; export { EvictionNotSupportedError } from '../util/errors/eviction-not-supported-error'; +export { IdempotencyCollisionError } from '../util/errors/idempotency-collision-error'; export { EvictionPolicy, DEFAULT_DURABLE_TASK_EVICTION_POLICY, diff --git a/sdks/typescript/src/v1/task.ts b/sdks/typescript/src/v1/task.ts index 86c7695ebc..4592fe31a6 100644 --- a/sdks/typescript/src/v1/task.ts +++ b/sdks/typescript/src/v1/task.ts @@ -42,6 +42,23 @@ export type Concurrency = { */ export type TaskConcurrency = Concurrency; +/** + * Configuration for idempotency on a workflow or standalone task. + * Prevents more than one run from occurring within a given time window. + */ +export type IdempotencyConfig = { + /** + * CEL expression to create an idempotency key from input and metadata. + * @example "input.id" // use the 'id' field from input as the key + */ + expression: string; + + /** + * How long the idempotency key should live (in milliseconds). + */ + ttlMs: number; +}; + export class NonRetryableError extends Error { constructor(message?: string) { super(message); diff --git a/sql/schema/v0.sql b/sql/schema/v0.sql index b1a04f58e7..64ba887e87 100644 --- a/sql/schema/v0.sql +++ b/sql/schema/v0.sql @@ -1088,6 +1088,8 @@ CREATE TABLE "defaultPriority" INTEGER, "createWorkflowVersionOpts" JSONB, "inputJsonSchema" JSONB, + "idempotencyKeyExpression" TEXT, + "idempotencyKeyTtlMs" BIGINT, CONSTRAINT "WorkflowVersion_pkey" PRIMARY KEY ("id") ); diff --git a/sql/schema/v1-olap.sql b/sql/schema/v1-olap.sql index 4df675c1b5..0e5b01a8a9 100644 --- a/sql/schema/v1-olap.sql +++ b/sql/schema/v1-olap.sql @@ -576,7 +576,7 @@ CREATE TABLE v1_event_to_run_olap ( PRIMARY KEY (event_id, event_seen_at, run_id, run_inserted_at) ) PARTITION BY RANGE(event_seen_at); -CREATE TYPE v1_cel_evaluation_failure_source AS ENUM ('FILTER', 'WEBHOOK'); +CREATE TYPE v1_cel_evaluation_failure_source AS ENUM ('FILTER', 'WEBHOOK', 'IDEMPOTENCY_KEY'); CREATE TABLE v1_cel_evaluation_failures_olap ( id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY,