diff --git a/api-contracts/v1/shared/trigger.proto b/api-contracts/v1/shared/trigger.proto index ad8913a43d..0c981d204a 100644 --- a/api-contracts/v1/shared/trigger.proto +++ b/api-contracts/v1/shared/trigger.proto @@ -74,4 +74,7 @@ message TriggerWorkflowRequest { // (optional) the desired worker labels for the workflow run, which will be used to determine which workers can pick up the workflow's tasks. if not set, defaults to an empty set of labels, which means any worker can pick up the tasks. map desired_worker_labels = 10; + + // (optional) idempotency key for deduplicating workflow runs + optional string idempotency_key = 11; } diff --git a/api-contracts/v1/workflows.proto b/api-contracts/v1/workflows.proto index e155664f61..681dc68133 100644 --- a/api-contracts/v1/workflows.proto +++ b/api-contracts/v1/workflows.proto @@ -50,6 +50,8 @@ message TriggerWorkflowRunRequest { bytes additional_metadata = 3; optional int32 priority = 4; map desired_worker_labels = 5; + // (optional) idempotency key for deduplicating workflow runs + optional string idempotency_key = 6; } message TriggerWorkflowRunResponse { diff --git a/cmd/hatchet-migrate/migrate/migrations/20260205120000_v1_0_76.sql b/cmd/hatchet-migrate/migrate/migrations/20260205120000_v1_0_76.sql new file mode 100644 index 0000000000..a40f499ca0 --- /dev/null +++ b/cmd/hatchet-migrate/migrate/migrations/20260205120000_v1_0_76.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE v1_idempotency_key + ADD COLUMN last_denied_at TIMESTAMPTZ; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE v1_idempotency_key + DROP COLUMN last_denied_at; +-- +goose StatementEnd diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..462577601f --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1751274312, + "narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1770197578, + "narHash": "sha256-AYqlWrX09+HvGs8zM6ebZ1pwUqjkfpnv8mewYwAo+iM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..ba3f49cdb6 --- /dev/null +++ b/flake.nix @@ -0,0 +1,38 @@ +{ + description = "Hatchet development shell"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + pkgsUnstable = import nixpkgs-unstable { inherit system; }; + go = pkgs.go; + in + { + devShells.default = pkgs.mkShell { + packages = [ + go + pkgs.gopls + pkgs.gotools + pkgsUnstable.golangci-lint + pkgs.go-task + pkgs.protobuf + pkgs.protoc-gen-go + pkgs.protoc-gen-go-grpc + pkgs.nodejs_20 + pkgs.pnpm + pkgs.python312 + pkgs.poetry + pkgs.gcc + pkgs.pkg-config + pkgs.git + ]; + }; + }); +} diff --git a/internal/services/admin/contracts/workflows.pb.go b/internal/services/admin/contracts/workflows.pb.go index d24f45f015..278a4d0df1 100644 --- a/internal/services/admin/contracts/workflows.pb.go +++ b/internal/services/admin/contracts/workflows.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v5.29.6 +// protoc v7.34.1 // source: workflows.proto package contracts diff --git a/internal/services/admin/contracts/workflows_grpc.pb.go b/internal/services/admin/contracts/workflows_grpc.pb.go index 7e000f1df5..2a64608209 100644 --- a/internal/services/admin/contracts/workflows_grpc.pb.go +++ b/internal/services/admin/contracts/workflows_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v5.29.6 +// - protoc v7.34.1 // source: workflows.proto package contracts diff --git a/internal/services/admin/server_v1.go b/internal/services/admin/server_v1.go index 7a38a46eba..fb428472c0 100644 --- a/internal/services/admin/server_v1.go +++ b/internal/services/admin/server_v1.go @@ -223,9 +223,16 @@ func (i *AdminServiceImpl) newTriggerOpt( return nil, fmt.Errorf("could not create trigger opt: %w", err) } - return &v1.WorkflowNameTriggerOpts{ + opts := &v1.WorkflowNameTriggerOpts{ TriggerTaskData: t, - }, nil + } + + if req.IdempotencyKey != nil { + key := v1.IdempotencyKey(*req.IdempotencyKey) + opts.IdempotencyKey = &key + } + + return opts, nil } func (i *AdminServiceImpl) generateExternalIds(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts) error { @@ -258,6 +265,10 @@ func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts // if we have a scheduling error, we'll fall back to normal ingestion if schedulingErr != nil { + if errors.Is(schedulingErr, v1.ErrIdempotencyKeyAlreadyClaimed) { + return status.Error(codes.AlreadyExists, schedulingErr.Error()) + } + if !errors.Is(schedulingErr, schedulingv1.ErrTenantNotFound) && !errors.Is(schedulingErr, schedulingv1.ErrNoOptimisticSlots) { i.l.Error().Ctx(ctx).Err(schedulingErr).Msg("could not run optimistic scheduling") } @@ -297,8 +308,14 @@ func (i *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts triggerErr := i.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) // if we fail to trigger via gRPC, we fall back to normal ingestion - if triggerErr != nil && !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { - i.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") + if triggerErr != nil { + if errors.Is(triggerErr, v1.ErrIdempotencyKeyAlreadyClaimed) { + return status.Error(codes.AlreadyExists, triggerErr.Error()) + } + + if !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { + i.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") + } } else if triggerErr == nil { return nil } diff --git a/internal/services/admin/v1/server.go b/internal/services/admin/v1/server.go index 6873d51c2c..bf21b1bfd6 100644 --- a/internal/services/admin/v1/server.go +++ b/internal/services/admin/v1/server.go @@ -578,9 +578,16 @@ func (a *AdminServiceImpl) newTriggerOpt( t.Priority = req.Priority } - return &v1.WorkflowNameTriggerOpts{ + opts := &v1.WorkflowNameTriggerOpts{ TriggerTaskData: t, - }, nil + } + + if req.IdempotencyKey != nil { + key := v1.IdempotencyKey(*req.IdempotencyKey) + opts.IdempotencyKey = &key + } + + return opts, nil } func (a *AdminServiceImpl) generateExternalIds(ctx context.Context, tenantId uuid.UUID, opts []*v1.WorkflowNameTriggerOpts) error { @@ -613,6 +620,10 @@ func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts // if we have a scheduling error, we'll fall back to normal ingestion if schedulingErr != nil { + if errors.Is(schedulingErr, v1.ErrIdempotencyKeyAlreadyClaimed) { + return status.Error(codes.AlreadyExists, schedulingErr.Error()) + } + if !errors.Is(schedulingErr, schedulingv1.ErrTenantNotFound) && !errors.Is(schedulingErr, schedulingv1.ErrNoOptimisticSlots) { a.l.Error().Ctx(ctx).Err(schedulingErr).Msg("could not run optimistic scheduling") } @@ -652,8 +663,14 @@ func (a *AdminServiceImpl) ingest(ctx context.Context, tenantId uuid.UUID, opts triggerErr := a.tw.TriggerFromWorkflowNames(ctx, tenantId, optsToSend) // if we fail to trigger via gRPC, we fall back to normal ingestion - if triggerErr != nil && !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { - a.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") + if triggerErr != nil { + if errors.Is(triggerErr, v1.ErrIdempotencyKeyAlreadyClaimed) { + return status.Error(codes.AlreadyExists, triggerErr.Error()) + } + + if !errors.Is(triggerErr, trigger.ErrNoTriggerSlots) { + a.l.Error().Ctx(ctx).Err(triggerErr).Msg("could not trigger workflow runs via gRPC") + } } else if triggerErr == nil { return nil } diff --git a/internal/services/controllers/olap/process_dag_status_updates.go b/internal/services/controllers/olap/process_dag_status_updates.go index e364eb3169..f610c6974b 100644 --- a/internal/services/controllers/olap/process_dag_status_updates.go +++ b/internal/services/controllers/olap/process_dag_status_updates.go @@ -53,6 +53,14 @@ func (o *OLAPControllerImpl) notifyDAGsUpdated(ctx context.Context, rows []v1.Up tenantIdToPayloads := make(map[uuid.UUID][]tasktypes.NotifyFinalizedPayload) for _, row := range rows { + if row.ReadableStatus == sqlcv1.V1ReadableStatusOlapCOMPLETED || + row.ReadableStatus == sqlcv1.V1ReadableStatusOlapCANCELLED || + row.ReadableStatus == sqlcv1.V1ReadableStatusOlapFAILED { + if err := o.repo.Idempotency().DeleteIdempotencyKeysByExternalId(ctx, row.TenantId, row.ExternalId); err != nil { + o.l.Error().Err(err).Str("dagExternalId", row.ExternalId.String()).Msg("failed to delete idempotency key for dag") + } + } + tenantIdToPayloads[row.TenantId] = append(tenantIdToPayloads[row.TenantId], tasktypes.NotifyFinalizedPayload{ ExternalId: row.ExternalId, Status: row.ReadableStatus, diff --git a/internal/services/controllers/olap/process_task_status_updates.go b/internal/services/controllers/olap/process_task_status_updates.go index 82d511cfdf..0f444d8fc0 100644 --- a/internal/services/controllers/olap/process_task_status_updates.go +++ b/internal/services/controllers/olap/process_task_status_updates.go @@ -56,6 +56,12 @@ func (o *OLAPControllerImpl) notifyTasksUpdated(ctx context.Context, rows []v1.U continue } + if !row.IsDAGTask { + if err := o.repo.Idempotency().DeleteIdempotencyKeysByExternalId(ctx, row.TenantId, row.ExternalId); err != nil { + o.l.Error().Err(err).Str("taskExternalId", row.ExternalId.String()).Msg("failed to delete idempotency key for task") + } + } + tenantIdToPayloads[row.TenantId] = append(tenantIdToPayloads[row.TenantId], tasktypes.NotifyFinalizedPayload{ ExternalId: row.ExternalId, Status: row.ReadableStatus, diff --git a/internal/services/controllers/task/controller.go b/internal/services/controllers/task/controller.go index 98b9fb187e..7a134c94bb 100644 --- a/internal/services/controllers/task/controller.go +++ b/internal/services/controllers/task/controller.go @@ -1100,7 +1100,19 @@ func (tc *TasksControllerImpl) handleProcessInternalEvents(ctx context.Context, // handleProcessEventTrigger is responsible for inserting tasks into the database based on event triggers. func (tc *TasksControllerImpl) handleProcessTaskTrigger(ctx context.Context, tenantId uuid.UUID, payloads [][]byte) error { - return tc.tw.TriggerFromWorkflowNames(ctx, tenantId, msgqueue.JSONConvert[v1.WorkflowNameTriggerOpts](payloads)) + opts := msgqueue.JSONConvert[v1.WorkflowNameTriggerOpts](payloads) + for _, opt := range opts { + opt.AllowPartialIdempotency = true + } + + err := tc.tw.TriggerFromWorkflowNames(ctx, tenantId, opts) + + if errors.Is(err, v1.ErrIdempotencyKeyAlreadyClaimed) { + tc.l.Debug().Err(err).Msg("skipping workflow trigger with duplicate idempotency key") + return nil + } + + return err } // processUserEventMatches looks for user event matches diff --git a/internal/services/shared/proto/v1/trigger.pb.go b/internal/services/shared/proto/v1/trigger.pb.go index 73fdab6836..a8e3096882 100644 --- a/internal/services/shared/proto/v1/trigger.pb.go +++ b/internal/services/shared/proto/v1/trigger.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v5.29.6 +// protoc v7.34.1 // source: v1/shared/trigger.proto package v1 @@ -198,6 +198,8 @@ type TriggerWorkflowRequest struct { Priority *int32 `protobuf:"varint,9,opt,name=priority,proto3,oneof" json:"priority,omitempty"` // (optional) the desired worker labels for the workflow run, which will be used to determine which workers can pick up the workflow's tasks. if not set, defaults to an empty set of labels, which means any worker can pick up the tasks. DesiredWorkerLabels map[string]*DesiredWorkerLabels `protobuf:"bytes,10,rep,name=desired_worker_labels,json=desiredWorkerLabels,proto3" json:"desired_worker_labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // (optional) idempotency key for deduplicating workflow runs + IdempotencyKey *string `protobuf:"bytes,11,opt,name=idempotency_key,json=idempotencyKey,proto3,oneof" json:"idempotency_key,omitempty"` } func (x *TriggerWorkflowRequest) Reset() { @@ -302,6 +304,13 @@ func (x *TriggerWorkflowRequest) GetDesiredWorkerLabels() map[string]*DesiredWor return nil } +func (x *TriggerWorkflowRequest) GetIdempotencyKey() string { + if x != nil && x.IdempotencyKey != nil { + return *x.IdempotencyKey + } + return "" +} + var File_v1_shared_trigger_proto protoreflect.FileDescriptor var file_v1_shared_trigger_proto_rawDesc = []byte{ @@ -324,7 +333,7 @@ var file_v1_shared_trigger_proto_rawDesc = []byte{ 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x42, - 0x09, 0x0a, 0x07, 0x5f, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xc8, 0x05, 0x0a, 0x16, 0x54, + 0x09, 0x0a, 0x07, 0x5f, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8a, 0x06, 0x0a, 0x16, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, @@ -354,35 +363,39 @@ var file_v1_shared_trigger_proto_rawDesc = []byte{ 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, - 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x5f, 0x0a, - 0x18, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, - 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, - 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, - 0x0a, 0x0a, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x1e, 0x0a, 0x1c, - 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x72, 0x75, 0x6e, - 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, - 0x5f, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x0c, 0x0a, 0x0a, - 0x5f, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, - 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x77, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x69, - 0x6f, 0x72, 0x69, 0x74, 0x79, 0x2a, 0x85, 0x01, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, - 0x09, 0x0a, 0x05, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, - 0x54, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x47, 0x52, 0x45, - 0x41, 0x54, 0x45, 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x47, - 0x52, 0x45, 0x41, 0x54, 0x45, 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x5f, 0x4f, 0x52, 0x5f, 0x45, - 0x51, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, - 0x48, 0x41, 0x4e, 0x10, 0x04, 0x12, 0x16, 0x0a, 0x12, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x48, - 0x41, 0x4e, 0x5f, 0x4f, 0x52, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x05, 0x42, 0x42, 0x5a, - 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x74, 0x63, - 0x68, 0x65, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2f, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, - 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x2c, 0x0a, + 0x0f, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x07, 0x52, 0x0e, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, + 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x1a, 0x5f, 0x0a, 0x18, 0x44, + 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, + 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, + 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, + 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, 0x64, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, + 0x6e, 0x63, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x2a, 0x85, 0x01, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, + 0x72, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, + 0x4e, 0x4f, 0x54, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x47, + 0x52, 0x45, 0x41, 0x54, 0x45, 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x10, 0x02, 0x12, 0x19, 0x0a, + 0x15, 0x47, 0x52, 0x45, 0x41, 0x54, 0x45, 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x5f, 0x4f, 0x52, + 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x4c, 0x45, 0x53, 0x53, + 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x10, 0x04, 0x12, 0x16, 0x0a, 0x12, 0x4c, 0x45, 0x53, 0x53, 0x5f, + 0x54, 0x48, 0x41, 0x4e, 0x5f, 0x4f, 0x52, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x05, 0x42, + 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, + 0x74, 0x63, 0x68, 0x65, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x74, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/services/shared/proto/v1/workflows.pb.go b/internal/services/shared/proto/v1/workflows.pb.go index db2a08f1e7..1b12ce6ae4 100644 --- a/internal/services/shared/proto/v1/workflows.pb.go +++ b/internal/services/shared/proto/v1/workflows.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.1 -// protoc v5.29.6 +// protoc v7.34.1 // source: v1/workflows.proto package v1 @@ -534,6 +534,8 @@ type TriggerWorkflowRunRequest struct { AdditionalMetadata []byte `protobuf:"bytes,3,opt,name=additional_metadata,json=additionalMetadata,proto3" json:"additional_metadata,omitempty"` Priority *int32 `protobuf:"varint,4,opt,name=priority,proto3,oneof" json:"priority,omitempty"` DesiredWorkerLabels map[string]*DesiredWorkerLabels `protobuf:"bytes,5,rep,name=desired_worker_labels,json=desiredWorkerLabels,proto3" json:"desired_worker_labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // (optional) idempotency key for deduplicating workflow runs + IdempotencyKey *string `protobuf:"bytes,6,opt,name=idempotency_key,json=idempotencyKey,proto3,oneof" json:"idempotency_key,omitempty"` } func (x *TriggerWorkflowRunRequest) Reset() { @@ -603,6 +605,13 @@ func (x *TriggerWorkflowRunRequest) GetDesiredWorkerLabels() map[string]*Desired return nil } +func (x *TriggerWorkflowRunRequest) GetIdempotencyKey() string { + if x != nil && x.IdempotencyKey != nil { + return *x.IdempotencyKey + } + return "" +} + type TriggerWorkflowRunResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1626,7 +1635,7 @@ var file_v1_workflows_proto_rawDesc = []byte{ 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x54, 0x61, 0x73, - 0x6b, 0x73, 0x22, 0x82, 0x03, 0x0a, 0x19, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, + 0x6b, 0x73, 0x22, 0xc4, 0x03, 0x0a, 0x19, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, @@ -1643,275 +1652,279 @@ var file_v1_workflows_proto_rawDesc = []byte{ 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x5f, 0x0a, 0x18, 0x44, 0x65, 0x73, - 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x69, - 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, - 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x22, 0x3d, 0x0a, 0x1a, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0x7a, 0x0a, 0x18, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, - 0x73, 0x6b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, - 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, - 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, - 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x49, 0x64, 0x22, 0x7b, 0x0a, 0x19, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x44, 0x75, 0x72, 0x61, - 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x45, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, - 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x64, 0x22, - 0xdd, 0x05, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, - 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x72, 0x6f, 0x6e, 0x5f, - 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x63, 0x72, 0x6f, 0x6e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x28, 0x0a, 0x05, - 0x74, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x52, - 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x31, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0b, 0x63, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0a, 0x63, 0x72, 0x6f, - 0x6e, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x09, 0x63, 0x72, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, - 0x0f, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x48, 0x01, 0x52, 0x0d, 0x6f, 0x6e, - 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x88, 0x01, 0x01, 0x12, 0x2f, - 0x0a, 0x06, 0x73, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, - 0x67, 0x79, 0x48, 0x02, 0x52, 0x06, 0x73, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x88, 0x01, 0x01, 0x12, - 0x2e, 0x0a, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, - 0x69, 0x74, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x48, 0x03, 0x52, 0x0f, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x88, 0x01, 0x01, 0x12, - 0x38, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x61, - 0x72, 0x72, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x41, 0x72, 0x72, 0x12, 0x3a, 0x0a, 0x0f, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x11, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x6a, - 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0c, - 0x48, 0x04, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x72, 0x6f, 0x6e, 0x5f, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, - 0x69, 0x63, 0x6b, 0x79, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, - 0x70, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x22, 0xb7, 0x01, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, - 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x1e, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x75, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x52, 0x75, 0x6e, 0x73, 0x88, 0x01, - 0x01, 0x12, 0x48, 0x0a, 0x0e, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, - 0x65, 0x67, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, - 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x48, 0x01, 0x52, 0x0d, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, - 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x75, 0x6e, 0x73, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x22, 0x85, 0x07, 0x0a, 0x0e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x12, 0x1f, - 0x0a, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, - 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x38, 0x0a, - 0x0b, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, - 0x73, 0x6b, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x52, 0x0a, 0x72, 0x61, 0x74, - 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x49, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x5f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, - 0x74, 0x73, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, - 0x6c, 0x73, 0x12, 0x2a, 0x0a, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x66, 0x61, - 0x63, 0x74, 0x6f, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x0d, 0x62, 0x61, - 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x33, - 0x0a, 0x13, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x11, 0x62, - 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x61, 0x78, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x63, 0x79, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, - 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x02, - 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x88, 0x01, 0x01, 0x12, - 0x2e, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x0f, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, - 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x49, - 0x0a, 0x0d, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, - 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x2e, 0x53, 0x6c, 0x6f, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x73, 0x6c, 0x6f, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x1a, 0x58, 0x0a, 0x11, 0x57, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x6c, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, - 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x62, 0x61, 0x63, 0x6b, - 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x42, - 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x13, - 0x0a, 0x11, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x22, 0xb8, 0x02, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, - 0x73, 0x6b, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x19, 0x0a, - 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x05, - 0x75, 0x6e, 0x69, 0x74, 0x73, 0x88, 0x01, 0x01, 0x12, 0x1e, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, - 0x65, 0x78, 0x70, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x6b, 0x65, - 0x79, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, 0x22, 0x0a, 0x0a, 0x75, 0x6e, 0x69, 0x74, - 0x73, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x09, - 0x75, 0x6e, 0x69, 0x74, 0x73, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2f, 0x0a, 0x11, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x65, 0x78, 0x70, - 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x0f, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, 0x36, 0x0a, - 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x15, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x44, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x04, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x42, - 0x0b, 0x0a, 0x09, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x42, 0x0d, 0x0a, 0x0b, - 0x5f, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x42, 0x14, 0x0a, 0x12, 0x5f, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x65, 0x78, 0x70, - 0x72, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x50, - 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, - 0x22, 0x37, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0xe4, 0x01, 0x0a, 0x0d, 0x54, 0x61, - 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x1b, - 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x01, - 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x0b, 0x72, - 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x69, 0x73, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x69, 0x73, 0x45, 0x76, 0x69, 0x63, 0x74, 0x65, 0x64, 0x42, 0x08, 0x0a, 0x06, 0x5f, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x22, 0xce, 0x02, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x12, 0x25, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x09, 0x74, 0x61, 0x73, 0x6b, 0x5f, - 0x72, 0x75, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x08, 0x74, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, - 0x65, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, - 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, 0x65, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, 0x76, 0x69, 0x63, 0x74, 0x65, - 0x64, 0x1a, 0x4e, 0x0a, 0x0d, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x2a, 0x24, 0x0a, 0x0e, 0x53, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x53, 0x74, 0x72, 0x61, 0x74, - 0x65, 0x67, 0x79, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x4f, 0x46, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, - 0x04, 0x48, 0x41, 0x52, 0x44, 0x10, 0x01, 0x2a, 0x5d, 0x0a, 0x11, 0x52, 0x61, 0x74, 0x65, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, - 0x53, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x49, 0x4e, 0x55, - 0x54, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, 0x02, 0x12, 0x07, - 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x45, 0x45, 0x4b, 0x10, - 0x04, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x4f, 0x4e, 0x54, 0x48, 0x10, 0x05, 0x12, 0x08, 0x0a, 0x04, - 0x59, 0x45, 0x41, 0x52, 0x10, 0x06, 0x2a, 0x5b, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x0a, 0x0a, 0x06, 0x51, 0x55, 0x45, 0x55, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, - 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x46, - 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x43, 0x45, - 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x56, 0x49, 0x43, 0x54, 0x45, - 0x44, 0x10, 0x05, 0x2a, 0x7f, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, - 0x16, 0x0a, 0x12, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x49, 0x4e, 0x5f, 0x50, 0x52, 0x4f, - 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x52, 0x4f, 0x50, 0x5f, - 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x51, 0x55, 0x45, 0x55, - 0x45, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x47, 0x52, - 0x4f, 0x55, 0x50, 0x5f, 0x52, 0x4f, 0x55, 0x4e, 0x44, 0x5f, 0x52, 0x4f, 0x42, 0x49, 0x4e, 0x10, - 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x4e, 0x45, 0x57, 0x45, - 0x53, 0x54, 0x10, 0x04, 0x32, 0xcf, 0x03, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x52, 0x0a, 0x0b, 0x50, 0x75, 0x74, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x20, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, + 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x2c, 0x0a, 0x0f, 0x69, 0x64, 0x65, + 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x01, 0x52, 0x0e, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, + 0x79, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x1a, 0x5f, 0x0a, 0x18, 0x44, 0x65, 0x73, 0x69, 0x72, + 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, + 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, + 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x0a, 0x1a, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0x7a, 0x0a, 0x18, 0x42, 0x72, 0x61, 0x6e, + 0x63, 0x68, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x74, 0x61, 0x73, 0x6b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x17, + 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x72, 0x61, 0x6e, 0x63, + 0x68, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, + 0x63, 0x68, 0x49, 0x64, 0x22, 0x7b, 0x0a, 0x19, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x44, 0x75, + 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x73, + 0x6b, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6e, + 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x6f, + 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, + 0x64, 0x22, 0xdd, 0x05, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x72, 0x69, 0x67, + 0x67, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x72, 0x6f, + 0x6e, 0x5f, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x63, 0x72, 0x6f, 0x6e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x28, + 0x0a, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, + 0x73, 0x52, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x31, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, + 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0b, + 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0a, 0x63, + 0x72, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x09, 0x63, 0x72, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, + 0x3f, 0x0a, 0x0f, 0x6f, 0x6e, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, + 0x73, 0x6b, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x48, 0x01, 0x52, 0x0d, + 0x6f, 0x6e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x88, 0x01, 0x01, + 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x53, 0x74, 0x72, 0x61, + 0x74, 0x65, 0x67, 0x79, 0x48, 0x02, 0x52, 0x06, 0x73, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x88, 0x01, + 0x01, 0x12, 0x2e, 0x0a, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x48, 0x03, 0x52, 0x0f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x88, 0x01, + 0x01, 0x12, 0x38, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, + 0x5f, 0x61, 0x72, 0x72, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0e, 0x63, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x41, 0x72, 0x72, 0x12, 0x3a, 0x0a, 0x0f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x11, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x0c, 0x48, 0x04, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x72, 0x6f, + 0x6e, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6f, 0x6e, 0x5f, 0x66, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x73, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, 0x14, 0x0a, 0x12, 0x5f, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x22, 0x70, 0x0a, 0x0d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x22, 0xb7, 0x01, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x63, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x75, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x52, 0x75, 0x6e, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x48, 0x0a, 0x0e, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x72, + 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, + 0x74, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x48, 0x01, 0x52, 0x0d, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, + 0x09, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x75, 0x6e, 0x73, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x22, 0x85, 0x07, + 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, + 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, + 0x64, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, + 0x38, 0x0a, 0x0b, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x07, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x54, 0x61, 0x73, 0x6b, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x52, 0x0a, 0x72, + 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x49, 0x0a, 0x0d, 0x77, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x5f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, + 0x4f, 0x70, 0x74, 0x73, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x73, 0x12, 0x2a, 0x0a, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, + 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x0d, + 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x88, 0x01, 0x01, + 0x12, 0x33, 0x0a, 0x13, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x61, 0x78, 0x5f, + 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, + 0x11, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x61, 0x78, 0x53, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x73, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x63, 0x79, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0b, 0x63, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x48, 0x02, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x88, 0x01, + 0x01, 0x12, 0x2e, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x0f, 0x73, + 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, + 0x01, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x18, + 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, + 0x12, 0x49, 0x0a, 0x0d, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4f, 0x70, 0x74, 0x73, 0x2e, 0x53, 0x6c, 0x6f, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x73, + 0x6c, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x1a, 0x58, 0x0a, 0x11, 0x57, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x53, 0x6c, 0x6f, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, + 0x66, 0x66, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x62, 0x61, + 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0xb8, 0x02, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x54, 0x61, 0x73, 0x6b, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x19, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, + 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x88, 0x01, 0x01, 0x12, 0x1e, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, + 0x6b, 0x65, 0x79, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, 0x22, 0x0a, 0x0a, 0x75, 0x6e, + 0x69, 0x74, 0x73, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, + 0x52, 0x09, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2f, + 0x0a, 0x11, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x65, + 0x78, 0x70, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x0f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x45, 0x78, 0x70, 0x72, 0x88, 0x01, 0x01, 0x12, + 0x36, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x15, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x04, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x75, 0x6e, 0x69, 0x74, + 0x73, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x42, 0x0d, + 0x0a, 0x0b, 0x5f, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x42, 0x14, 0x0a, + 0x12, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x65, + 0x78, 0x70, 0x72, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x22, 0x50, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x49, 0x64, 0x22, 0x37, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0xe4, 0x01, 0x0a, 0x0d, + 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x1f, 0x0a, + 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x25, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, + 0x12, 0x1b, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x48, 0x01, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, + 0x0b, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, 0x76, 0x69, 0x63, 0x74, 0x65, 0x64, 0x42, 0x08, 0x0a, + 0x06, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x22, 0xce, 0x02, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x12, 0x25, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x09, 0x74, 0x61, 0x73, + 0x6b, 0x5f, 0x72, 0x75, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x74, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, 0x12, + 0x12, 0x0a, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, + 0x6f, 0x6e, 0x65, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x12, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, + 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, 0x76, 0x69, 0x63, + 0x74, 0x65, 0x64, 0x1a, 0x4e, 0x0a, 0x0d, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x75, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, + 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x2a, 0x24, 0x0a, 0x0e, 0x53, 0x74, 0x69, 0x63, 0x6b, 0x79, 0x53, 0x74, 0x72, + 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x4f, 0x46, 0x54, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x48, 0x41, 0x52, 0x44, 0x10, 0x01, 0x2a, 0x5d, 0x0a, 0x11, 0x52, 0x61, 0x74, + 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, + 0x0a, 0x06, 0x53, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x49, + 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, 0x02, + 0x12, 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x45, 0x45, + 0x4b, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x4f, 0x4e, 0x54, 0x48, 0x10, 0x05, 0x12, 0x08, + 0x0a, 0x04, 0x59, 0x45, 0x41, 0x52, 0x10, 0x06, 0x2a, 0x5b, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0a, 0x0a, 0x06, 0x51, 0x55, 0x45, 0x55, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, + 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0a, 0x0a, + 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, + 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x56, 0x49, 0x43, + 0x54, 0x45, 0x44, 0x10, 0x05, 0x2a, 0x7f, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, + 0x79, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x49, 0x4e, 0x5f, 0x50, + 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x52, 0x4f, + 0x50, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x51, 0x55, + 0x45, 0x55, 0x45, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, + 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x52, 0x4f, 0x55, 0x4e, 0x44, 0x5f, 0x52, 0x4f, 0x42, 0x49, + 0x4e, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x4e, 0x45, + 0x57, 0x45, 0x53, 0x54, 0x10, 0x04, 0x32, 0xcf, 0x03, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x52, 0x0a, 0x0b, 0x50, 0x75, 0x74, 0x57, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x20, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x61, 0x6e, - 0x63, 0x65, 0x6c, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, - 0x6e, 0x63, 0x65, 0x6c, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x54, 0x61, 0x73, 0x6b, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x52, 0x65, 0x70, - 0x6c, 0x61, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x70, 0x6c, 0x61, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x54, 0x61, 0x73, 0x6b, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x12, 0x54, 0x72, 0x69, - 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x12, - 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, - 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x11, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x44, 0x75, - 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x42, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x43, + 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x54, 0x61, + 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x52, + 0x65, 0x70, 0x6c, 0x61, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x54, 0x61, + 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x12, 0x54, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, + 0x6e, 0x12, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x44, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x11, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, + 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x1c, 0x2e, 0x76, 0x31, + 0x2e, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, + 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x72, 0x61, - 0x6e, 0x63, 0x68, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2d, 0x64, 0x65, 0x76, - 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x68, 0x61, 0x72, 0x65, - 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2d, 0x64, + 0x65, 0x76, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x68, 0x61, + 0x72, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/services/shared/proto/v1/workflows_grpc.pb.go b/internal/services/shared/proto/v1/workflows_grpc.pb.go index 143c1c06e6..fd7d38e453 100644 --- a/internal/services/shared/proto/v1/workflows_grpc.pb.go +++ b/internal/services/shared/proto/v1/workflows_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 -// - protoc v5.29.6 +// - protoc v7.34.1 // source: v1/workflows.proto package v1 diff --git a/pkg/client/admin.go b/pkg/client/admin.go index 20c9edcb17..570ef1ba77 100644 --- a/pkg/client/admin.go +++ b/pkg/client/admin.go @@ -354,6 +354,14 @@ func WithDesiredWorkerLabels(labels map[string]*types.DesiredWorkerLabel) RunOpt } } +func WithIdempotencyKey(key string) RunOptFunc { + return func(r *admincontracts.TriggerWorkflowRequest) error { + r.IdempotencyKey = &key + + return nil + } +} + func desiredWorkerLabelsToProto(labels map[string]*types.DesiredWorkerLabel) map[string]*admincontracts.DesiredWorkerLabels { if labels == nil { return nil @@ -387,7 +395,6 @@ func desiredWorkerLabelsToProto(labels map[string]*types.DesiredWorkerLabel) map return result } - // func WithSticky(sticky bool) RunOptFunc { // return func(r *admincontracts.TriggerWorkflowRequest) error { // r.Sticky = &sticky diff --git a/pkg/config/loader/loader.go b/pkg/config/loader/loader.go index 96f079eb79..fc7689fcef 100644 --- a/pkg/config/loader/loader.go +++ b/pkg/config/loader/loader.go @@ -407,6 +407,9 @@ func (c *ConfigLoader) InitDataLayer() (res *database.Layer, err error) { statusUpdateOpts, scf.Runtime.Limits, scf.Runtime.EnforceLimits, + scf.Runtime.EnableDurableUserEventLog, + scf.Runtime.IdempotencyKeyTTL, + scf.Runtime.IdempotencyKeyDenyRecheckInterval, ) if readReplicaPool != nil { diff --git a/pkg/config/server/server.go b/pkg/config/server/server.go index f4aaf91a7b..fd0f27eb0b 100644 --- a/pkg/config/server/server.go +++ b/pkg/config/server/server.go @@ -325,6 +325,17 @@ type ConfigFileRuntime struct { // TaskOperationLimits controls the limits for various task operations TaskOperationLimits TaskOperationLimitsConfigFile `mapstructure:"taskOperationLimits" json:"taskOperationLimits,omitempty"` + // EnableDurableUserEventLog controls whether we enable the durable event log for user events. By default, we don't persist user events + // to the core database, we only use them to trigger workflows. Enabling this will persist them to the core database. + EnableDurableUserEventLog bool `mapstructure:"enableDurableUserEventLog" json:"enableDurableUserEventLog,omitempty" default:"false"` + + // IdempotencyKeyTTL controls how long idempotency keys are retained as a safety net. Keys are deleted on completion, + // but this TTL limits how long a stuck key can block requeues. + IdempotencyKeyTTL time.Duration `mapstructure:"idempotencyKeyTTL" json:"idempotencyKeyTTL,omitempty" default:"24h"` + + // IdempotencyKeyDenyRecheckInterval controls how often we recheck terminal status for a claimed key + // before denying a duplicate enqueue. + IdempotencyKeyDenyRecheckInterval time.Duration `mapstructure:"idempotencyKeyDenyRecheckInterval" json:"idempotencyKeyDenyRecheckInterval,omitempty" default:"5m"` // WorkflowRunBufferSize is the buffer size for workflow run event batching in the dispatcher WorkflowRunBufferSize int `mapstructure:"workflowRunBufferSize" json:"workflowRunBufferSize,omitempty" default:"1000"` @@ -891,6 +902,10 @@ func BindAllEnv(v *viper.Viper) { _ = v.BindEnv("runtime.updateHashFactor", "SERVER_UPDATE_HASH_FACTOR") _ = v.BindEnv("runtime.updateConcurrentFactor", "SERVER_UPDATE_CONCURRENT_FACTOR") + // enable durable user event log + _ = v.BindEnv("runtime.enableDurableUserEventLog", "SERVER_ENABLE_DURABLE_USER_EVENT_LOG") + _ = v.BindEnv("runtime.idempotencyKeyTTL", "SERVER_IDEMPOTENCY_KEY_TTL") + _ = v.BindEnv("runtime.idempotencyKeyDenyRecheckInterval", "SERVER_IDEMPOTENCY_KEY_DENY_RECHECK_INTERVAL") // internal client options _ = v.BindEnv("internalClient.base.tlsStrategy", "SERVER_INTERNAL_CLIENT_BASE_STRATEGY") _ = v.BindEnv("internalClient.inheritBase", "SERVER_INTERNAL_CLIENT_BASE_INHERIT_BASE") diff --git a/pkg/repository/durable_events.go b/pkg/repository/durable_events.go index 20d6cdbc0e..5fd121bfa1 100644 --- a/pkg/repository/durable_events.go +++ b/pkg/repository/durable_events.go @@ -1296,7 +1296,7 @@ func (r *durableEventsRepository) IngestDurableTaskEvent(ctx context.Context, op } if len(newTriggerOpts) > 0 { - createdTasks, createdDags, triggerErr := r.triggerFromWorkflowNames(ctx, optTx, tenantId, newTriggerOpts) + createdTasks, createdDags, _, triggerErr := r.triggerFromWorkflowNames(ctx, optTx, tenantId, newTriggerOpts) if triggerErr != nil { return nil, fmt.Errorf("failed to trigger workflows: %w", triggerErr) diff --git a/pkg/repository/idempotency.go b/pkg/repository/idempotency.go index 6323986037..3c6dc0f3c4 100644 --- a/pkg/repository/idempotency.go +++ b/pkg/repository/idempotency.go @@ -14,6 +14,8 @@ type IdempotencyKey string type IdempotencyRepository interface { CreateIdempotencyKey(context context.Context, tenantId uuid.UUID, key string, expiresAt pgtype.Timestamptz) error + CreateIdempotencyKeys(context context.Context, tenantId uuid.UUID, keys []string, expiresAt pgtype.Timestamptz) error + DeleteIdempotencyKeysByExternalId(context context.Context, tenantId uuid.UUID, externalId uuid.UUID) error EvictExpiredIdempotencyKeys(context context.Context, tenantId uuid.UUID) error } @@ -35,6 +37,25 @@ func (r *idempotencyRepository) CreateIdempotencyKey(context context.Context, te }) } +func (r *idempotencyRepository) CreateIdempotencyKeys(context context.Context, tenantId uuid.UUID, keys []string, expiresAt pgtype.Timestamptz) error { + if len(keys) == 0 { + return nil + } + + return r.queries.CreateIdempotencyKeys(context, r.pool, sqlcv1.CreateIdempotencyKeysParams{ + Tenantid: tenantId, + Keys: keys, + Expiresat: expiresAt, + }) +} + +func (r *idempotencyRepository) DeleteIdempotencyKeysByExternalId(context context.Context, tenantId uuid.UUID, externalId uuid.UUID) error { + return r.queries.DeleteIdempotencyKeysByExternalId(context, r.pool, sqlcv1.DeleteIdempotencyKeysByExternalIdParams{ + Tenantid: tenantId, + Externalid: externalId, + }) +} + func (r *idempotencyRepository) EvictExpiredIdempotencyKeys(context context.Context, tenantId uuid.UUID) error { return r.queries.CleanUpExpiredIdempotencyKeys(context, r.pool, tenantId) } diff --git a/pkg/repository/idempotency_trigger_test.go b/pkg/repository/idempotency_trigger_test.go new file mode 100644 index 0000000000..474625b5e2 --- /dev/null +++ b/pkg/repository/idempotency_trigger_test.go @@ -0,0 +1,922 @@ +//go:build !e2e && !load && !rampup && !integration + +package repository + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/hatchet-dev/hatchet/pkg/config/limits" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" +) + +func setupRepositoryWithTTL(t *testing.T, pool *pgxpool.Pool, ttl time.Duration, recheckInterval time.Duration) (Repository, func()) { + t.Helper() + + logger := zerolog.Nop() + inlineStoreTTL := 24 * time.Hour + payloadStoreOpts := PayloadStoreRepositoryOpts{ + InlineStoreTTL: &inlineStoreTTL, + } + statusUpdateLimits := StatusUpdateBatchSizeLimits{ + Task: 1000, + DAG: 1000, + } + + repo, cleanup := NewRepository( + pool, + nil, + &logger, + 0, + 24*time.Hour, + 24*time.Hour, + 10, + TaskOperationLimits{}, + payloadStoreOpts, + statusUpdateLimits, + limits.LimitConfigFile{}, + false, + false, + ttl, + recheckInterval, + ) + + return repo, func() { + _ = cleanup() + } +} + +func setupTenant(t *testing.T, repo Repository, pool *pgxpool.Pool) uuid.UUID { + t.Helper() + ctx := context.Background() + queries := sqlcv1.New() + + _, err := queries.CreateControllerPartition(ctx, pool, pgtype.Text{String: "test", Valid: true}) + require.NoError(t, err) + + tenantId := uuid.New() + _, err = repo.Tenant().CreateTenant(ctx, &CreateTenantOpts{ + ID: &tenantId, + Name: "test-tenant", + Slug: "test-tenant-" + tenantId.String(), + }) + require.NoError(t, err) + + return tenantId +} + +type minimalWorkflow struct { + WorkflowID uuid.UUID + WorkflowVersionID uuid.UUID +} + +func createMinimalWorkflow(t *testing.T, pool *pgxpool.Pool, tenantId uuid.UUID, workflowName string) minimalWorkflow { + t.Helper() + ctx := context.Background() + queries := sqlcv1.New() + + workflowId := uuid.New() + _, err := queries.CreateWorkflow(ctx, pool, sqlcv1.CreateWorkflowParams{ + ID: workflowId, + CreatedAt: pgtype.Timestamp{}, + UpdatedAt: pgtype.Timestamp{}, + Deletedat: pgtype.Timestamp{}, + Tenantid: tenantId, + Name: workflowName, + Description: "test workflow", + }) + require.NoError(t, err) + + workflowVersionId := uuid.New() + _, err = queries.CreateWorkflowVersion(ctx, pool, sqlcv1.CreateWorkflowVersionParams{ + ID: workflowVersionId, + CreatedAt: pgtype.Timestamp{}, + UpdatedAt: pgtype.Timestamp{}, + Deletedat: pgtype.Timestamp{}, + Checksum: "test-checksum", + Version: pgtype.Text{String: "v1", Valid: true}, + Workflowid: workflowId, + Sticky: sqlcv1.NullStickyStrategy{}, + Kind: sqlcv1.NullWorkflowKind{}, + DefaultPriority: pgtype.Int4{}, + CreateWorkflowVersionOpts: []byte("{}"), + InputJsonSchema: []byte("{}"), + }) + require.NoError(t, err) + + jobId := uuid.New() + _, err = queries.CreateJob(ctx, pool, sqlcv1.CreateJobParams{ + ID: jobId, + CreatedAt: pgtype.Timestamp{}, + UpdatedAt: pgtype.Timestamp{}, + Deletedat: pgtype.Timestamp{}, + Tenantid: tenantId, + Workflowversionid: workflowVersionId, + Name: "job", + Description: "job description", + Kind: sqlcv1.NullJobKind{}, + }) + require.NoError(t, err) + + _, err = queries.UpsertAction(ctx, pool, sqlcv1.UpsertActionParams{ + Action: "test.action", + Tenantid: tenantId, + }) + require.NoError(t, err) + + _, err = queries.CreateStep(ctx, pool, sqlcv1.CreateStepParams{ + ID: uuid.New(), + CreatedAt: pgtype.Timestamp{}, + UpdatedAt: pgtype.Timestamp{}, + Deletedat: pgtype.Timestamp{}, + Readableid: "step-1", + Tenantid: tenantId, + Jobid: jobId, + Actionid: "test.action", + Timeout: pgtype.Text{}, + CustomUserData: []byte("{}"), + Retries: pgtype.Int4{}, + ScheduleTimeout: pgtype.Text{}, + }) + require.NoError(t, err) + + return minimalWorkflow{ + WorkflowID: workflowId, + WorkflowVersionID: workflowVersionId, + } +} + +func insertRunStatus(t *testing.T, pool *pgxpool.Pool, tenantId uuid.UUID, wf minimalWorkflow, externalId uuid.UUID, status sqlcv1.V1ReadableStatusOlap) { + t.Helper() + ctx := context.Background() + + insertedAt := time.Now().UTC() + runId := time.Now().UnixNano() + + _, err := pool.Exec(ctx, ` + INSERT INTO v1_runs_olap ( + tenant_id, + id, + inserted_at, + external_id, + readable_status, + kind, + workflow_id, + workflow_version_id, + additional_metadata, + parent_task_external_id + ) VALUES ($1, $2, $3, $4, $5, 'TASK', $6, $7, NULL, NULL) + `, tenantId, runId, insertedAt, externalId, status, wf.WorkflowID, wf.WorkflowVersionID) + require.NoError(t, err) + + _, err = pool.Exec(ctx, ` + INSERT INTO v1_lookup_table_olap ( + tenant_id, + external_id, + task_id, + dag_id, + inserted_at + ) VALUES ($1, $2, $3, NULL, $4) + `, tenantId, externalId, runId, insertedAt) + require.NoError(t, err) +} + +func insertTerminalTaskEventsForRun(t *testing.T, pool *pgxpool.Pool, tenantId uuid.UUID, externalId uuid.UUID) { + t.Helper() + ctx := context.Background() + + rows, err := pool.Query(ctx, ` + WITH lookup AS ( + SELECT external_id, dag_id, task_id, inserted_at + FROM v1_lookup_table + WHERE tenant_id = $1 AND external_id = $2 + ), dag_tasks AS ( + SELECT t.id, t.inserted_at, t.retry_count + FROM lookup l + JOIN v1_dag_to_task dt ON dt.dag_id = l.dag_id AND dt.dag_inserted_at = l.inserted_at + JOIN v1_task t ON t.id = dt.task_id AND t.inserted_at = dt.task_inserted_at + WHERE l.dag_id IS NOT NULL + ), task_only AS ( + SELECT t.id, t.inserted_at, t.retry_count + FROM lookup l + JOIN v1_task t ON t.id = l.task_id AND t.inserted_at = l.inserted_at + WHERE l.task_id IS NOT NULL + ) + SELECT id, inserted_at, retry_count FROM dag_tasks + UNION ALL + SELECT id, inserted_at, retry_count FROM task_only + `, tenantId, externalId) + require.NoError(t, err) + defer rows.Close() + + taskCount := 0 + for rows.Next() { + var ( + taskId int64 + taskInserted time.Time + retryCount int32 + ) + require.NoError(t, rows.Scan(&taskId, &taskInserted, &retryCount)) + taskCount++ + + _, err = pool.Exec(ctx, ` + INSERT INTO v1_task_event (tenant_id, task_id, task_inserted_at, retry_count, event_type) + VALUES ($1, $2, $3, $4, 'COMPLETED') + `, tenantId, taskId, taskInserted, retryCount) + require.NoError(t, err) + } + require.NoError(t, rows.Err()) + require.Greater(t, taskCount, 0) +} + +func getLastDeniedAt(t *testing.T, pool *pgxpool.Pool, tenantId uuid.UUID, key IdempotencyKey) pgtype.Timestamptz { + t.Helper() + ctx := context.Background() + + var lastDeniedAt pgtype.Timestamptz + err := pool.QueryRow(ctx, ` + SELECT last_denied_at + FROM v1_idempotency_key + WHERE tenant_id = $1 AND key = $2 + `, tenantId, string(key)).Scan(&lastDeniedAt) + require.NoError(t, err) + return lastDeniedAt +} + +func getClaimedByExternalId(t *testing.T, pool *pgxpool.Pool, tenantId uuid.UUID, key IdempotencyKey) *uuid.UUID { + t.Helper() + ctx := context.Background() + + var claimedByExternalId *uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT claimed_by_external_id + FROM v1_idempotency_key + WHERE tenant_id = $1 AND key = $2 + `, tenantId, string(key)).Scan(&claimedByExternalId) + require.NoError(t, err) + return claimedByExternalId +} + +func TestTriggerWorkflowIdempotency_DedupesActiveRun(t *testing.T) { + // Ensures a duplicate idempotency key is rejected while an active run holds the claim. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 2*time.Hour, 5*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key") + + opts1 := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{opts1}) + require.NoError(t, err) + + opts2 := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{opts2}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) +} + +func TestTriggerWorkflowIdempotency_AllowsAfterRelease(t *testing.T) { + // Shows that once a key is released (deleted), the same key can be used again. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 5*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-release") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + err = repo.Idempotency().DeleteIdempotencyKeysByExternalId(ctx, tenantId, first.ExternalId) + require.NoError(t, err) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.NoError(t, err) +} + +func TestTriggerWorkflowIdempotency_UsesConfiguredTTL(t *testing.T) { + // Validates that the configured TTL is applied to the stored idempotency key. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + ttl := 90 * time.Minute + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, ttl, 5*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-ttl") + + opts := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{opts}) + require.NoError(t, err) + + var expiresAt time.Time + err = pool.QueryRow(ctx, + `SELECT expires_at FROM v1_idempotency_key WHERE tenant_id = $1 AND key = $2`, + tenantId, + string(key), + ).Scan(&expiresAt) + require.NoError(t, err) + + expected := time.Now().Add(ttl) + assert.WithinDuration(t, expected, expiresAt, 2*time.Minute) +} + +func TestTriggerWorkflowIdempotency_ExpiredUnclaimedKeyIsReusable(t *testing.T) { + // An expired, unclaimed key should be refreshed and claimed instead of blocking. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-expired") + + _, err := pool.Exec(ctx, + `INSERT INTO v1_idempotency_key (tenant_id, key, expires_at) VALUES ($1, $2, $3)`, + tenantId, + string(key), + time.Now().Add(-1*time.Hour), + ) + require.NoError(t, err) + + opts := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{opts}) + require.NoError(t, err) + + claimed := getClaimedByExternalId(t, pool, tenantId, key) + require.NotNil(t, claimed) + assert.Equal(t, opts.ExternalId, *claimed) + + var expiresAt time.Time + err = pool.QueryRow(ctx, + `SELECT expires_at FROM v1_idempotency_key WHERE tenant_id = $1 AND key = $2`, + tenantId, + string(key), + ).Scan(&expiresAt) + require.NoError(t, err) + assert.True(t, expiresAt.After(time.Now())) +} + +func TestTriggerWorkflowIdempotency_ReclaimsAfterTerminalStatus(t *testing.T) { + // When OLAP reports terminal status, a previously claimed key is reclaimed and a new run is allowed. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-reclaim") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertRunStatus(t, pool, tenantId, wf, first.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.NoError(t, err) +} + +func TestTriggerWorkflowIdempotency_ReclaimDoesNotUpdateLastDeniedAt(t *testing.T) { + // Reclaimed keys should not have last_denied_at set since the enqueue succeeds. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-reclaim-denied") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertRunStatus(t, pool, tenantId, wf, first.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.NoError(t, err) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, key) + assert.False(t, lastDeniedAt.Valid) +} + +func TestTriggerWorkflowIdempotency_MixedKeysRejectsPartialByDefault(t *testing.T) { + // Public trigger paths should remain atomic: if any key is still duplicate, reject the whole batch. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + keyTerminal := IdempotencyKey("idem-key-terminal") + keyRunning := IdempotencyKey("idem-key-running") + + terminalFirst := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyTerminal, + } + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{terminalFirst}) + require.NoError(t, err) + insertRunStatus(t, pool, tenantId, wf, terminalFirst.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + runningFirst := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyRunning, + } + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{runningFirst}) + require.NoError(t, err) + insertRunStatus(t, pool, tenantId, wf, runningFirst.ExternalId, sqlcv1.V1ReadableStatusOlapRUNNING) + + terminalSecond := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyTerminal, + } + runningSecond := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyRunning, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{terminalSecond, runningSecond}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) + + claimedTerminal := getClaimedByExternalId(t, pool, tenantId, keyTerminal) + require.NotNil(t, claimedTerminal) + assert.Equal(t, terminalFirst.ExternalId, *claimedTerminal) + + claimedRunning := getClaimedByExternalId(t, pool, tenantId, keyRunning) + require.NotNil(t, claimedRunning) + assert.Equal(t, runningFirst.ExternalId, *claimedRunning) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, keyRunning) + assert.True(t, lastDeniedAt.Valid) + assert.WithinDuration(t, time.Now(), lastDeniedAt.Time, 2*time.Minute) +} + +func TestTriggerWorkflowIdempotency_MixedKeysAllowsPartialWhenEnabled(t *testing.T) { + // Internal queued trigger processing may skip duplicates while still creating non-duplicate runs. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + keyTerminal := IdempotencyKey("idem-key-terminal-allow-partial") + keyRunning := IdempotencyKey("idem-key-running-allow-partial") + + terminalFirst := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyTerminal, + } + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{terminalFirst}) + require.NoError(t, err) + insertRunStatus(t, pool, tenantId, wf, terminalFirst.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + runningFirst := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyRunning, + } + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{runningFirst}) + require.NoError(t, err) + insertRunStatus(t, pool, tenantId, wf, runningFirst.ExternalId, sqlcv1.V1ReadableStatusOlapRUNNING) + + terminalSecond := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyTerminal, + AllowPartialIdempotency: true, + } + runningSecond := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &keyRunning, + AllowPartialIdempotency: true, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{terminalSecond, runningSecond}) + require.NoError(t, err) + + claimedTerminal := getClaimedByExternalId(t, pool, tenantId, keyTerminal) + require.NotNil(t, claimedTerminal) + assert.Equal(t, terminalSecond.ExternalId, *claimedTerminal) + + claimedRunning := getClaimedByExternalId(t, pool, tenantId, keyRunning) + require.NotNil(t, claimedRunning) + assert.Equal(t, runningFirst.ExternalId, *claimedRunning) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, keyRunning) + assert.True(t, lastDeniedAt.Valid) + assert.WithinDuration(t, time.Now(), lastDeniedAt.Time, 2*time.Minute) +} + +func TestTriggerWorkflowIdempotency_ConcurrentDuplicateKey(t *testing.T) { + // Two concurrent triggers with the same key should yield one success and one duplicate error. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + key := IdempotencyKey("idem-key-concurrent") + + start := make(chan struct{}) + errs := make(chan error, 2) + + run := func(externalId uuid.UUID) { + <-start + opts := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: externalId, + IdempotencyKey: &key, + } + _, _, err := repo.Triggers().TriggerFromWorkflowNames(context.Background(), tenantId, []*WorkflowNameTriggerOpts{opts}) + errs <- err + } + + go run(uuid.New()) + go run(uuid.New()) + + close(start) + + var ( + successCount int + dupCount int + ) + + for i := 0; i < 2; i++ { + err := <-errs + if err == nil { + successCount++ + continue + } + if errors.Is(err, ErrIdempotencyKeyAlreadyClaimed) { + dupCount++ + } else { + t.Fatalf("unexpected error: %v", err) + } + } + + assert.Equal(t, 1, successCount) + assert.Equal(t, 1, dupCount) +} + +func TestTriggerWorkflowIdempotency_RecheckThrottled(t *testing.T) { + // If last_denied_at is recent, recheck is skipped and the duplicate is denied. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + recheckInterval := 2 * time.Hour + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, recheckInterval) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-throttle") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertRunStatus(t, pool, tenantId, wf, first.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + _, err = pool.Exec(ctx, `UPDATE v1_idempotency_key SET last_denied_at = NOW() WHERE tenant_id = $1 AND key = $2`, tenantId, string(key)) + require.NoError(t, err) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) +} + +func TestTriggerWorkflowIdempotency_RecheckDisabled(t *testing.T) { + // With recheck disabled, duplicates are denied and last_denied_at remains unset. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 0) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-no-recheck") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertRunStatus(t, pool, tenantId, wf, first.ExternalId, sqlcv1.V1ReadableStatusOlapFAILED) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, key) + assert.False(t, lastDeniedAt.Valid) +} + +func TestTriggerWorkflowIdempotency_RecheckMissingOlapUpdatesLastDeniedAt(t *testing.T) { + // If OLAP has no status row, duplicates should be denied and last_denied_at updated. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-missing-olap") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, key) + assert.True(t, lastDeniedAt.Valid) + assert.WithinDuration(t, time.Now(), lastDeniedAt.Time, 2*time.Minute) +} + +func TestTriggerWorkflowIdempotency_RecheckUsesCoreWhenOlapMissing(t *testing.T) { + // If OLAP is missing but core task events are terminal, we should reclaim and allow a new run. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + _ = createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-core-fallback") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertTerminalTaskEventsForRun(t, pool, tenantId, first.ExternalId) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.NoError(t, err) + + claimed := getClaimedByExternalId(t, pool, tenantId, key) + require.NotNil(t, claimed) + assert.Equal(t, second.ExternalId, *claimed) +} + +func TestTriggerWorkflowIdempotency_RecheckUpdatesLastDeniedAtWhenNonTerminal(t *testing.T) { + // If the run is non-terminal, we deny and update last_denied_at for throttling. + pool, cleanup := setupPostgresWithMigration(t) + defer cleanup() + + repo, cleanupRepo := setupRepositoryWithTTL(t, pool, 30*time.Minute, 1*time.Minute) + defer cleanupRepo() + + tenantId := setupTenant(t, repo, pool) + wf := createMinimalWorkflow(t, pool, tenantId, "test-workflow") + + ctx := context.Background() + key := IdempotencyKey("idem-key-non-terminal") + + first := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err := repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{first}) + require.NoError(t, err) + + insertRunStatus(t, pool, tenantId, wf, first.ExternalId, sqlcv1.V1ReadableStatusOlapRUNNING) + + second := &WorkflowNameTriggerOpts{ + TriggerTaskData: &TriggerTaskData{ + WorkflowName: "test-workflow", + }, + ExternalId: uuid.New(), + IdempotencyKey: &key, + } + + _, _, err = repo.Triggers().TriggerFromWorkflowNames(ctx, tenantId, []*WorkflowNameTriggerOpts{second}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIdempotencyKeyAlreadyClaimed) + + lastDeniedAt := getLastDeniedAt(t, pool, tenantId, key) + assert.True(t, lastDeniedAt.Valid) + assert.WithinDuration(t, time.Now(), lastDeniedAt.Time, 2*time.Minute) +} diff --git a/pkg/repository/ids.go b/pkg/repository/ids.go index c4dddd3103..e9b77f2cce 100644 --- a/pkg/repository/ids.go +++ b/pkg/repository/ids.go @@ -30,6 +30,11 @@ type WorkflowNameTriggerOpts struct { // (optional) The idempotency key to use for debouncing this task IdempotencyKey *IdempotencyKey + // AllowPartialIdempotency permits duplicate keys in the same batch to be skipped while + // still creating any non-duplicate runs. Public trigger APIs should keep this disabled + // so bulk requests remain atomic. + AllowPartialIdempotency bool + // Whether to skip the creation of the child workflow ShouldSkip bool } diff --git a/pkg/repository/main_test.go b/pkg/repository/main_test.go new file mode 100644 index 0000000000..1f57a53d66 --- /dev/null +++ b/pkg/repository/main_test.go @@ -0,0 +1,29 @@ +//go:build !e2e && !load && !rampup && !integration + +package repository + +import ( + "context" + "os" + "testing" +) + +func TestMain(m *testing.M) { + code := m.Run() + + if testPool != nil { + testPool.Close() + } + + if testContainer != nil { + _ = testContainer.Terminate(context.Background()) + } + + if originalDatabaseURL != "" { + _ = os.Setenv("DATABASE_URL", originalDatabaseURL) + } else { + _ = os.Unsetenv("DATABASE_URL") + } + + os.Exit(code) +} diff --git a/pkg/repository/olap.go b/pkg/repository/olap.go index a7edff63ea..897cd45146 100644 --- a/pkg/repository/olap.go +++ b/pkg/repository/olap.go @@ -323,11 +323,12 @@ func NewOLAPRepositoryFromPool( payloadStoreOpts PayloadStoreRepositoryOpts, statusUpdateBatchSizeLimits StatusUpdateBatchSizeLimits, cacheDuration time.Duration, + enableDurableUserEventLog bool, shouldPartitionOtelTables bool, ) (OLAPRepository, func() error) { v := validator.NewDefaultValidator() - shared, cleanupShared := newSharedRepository(pool, pool, v, l, payloadStoreOpts, tenantLimitConfig, enforceLimits, cacheDuration) + shared, cleanupShared := newSharedRepository(pool, pool, v, l, payloadStoreOpts, tenantLimitConfig, enforceLimits, cacheDuration, enableDurableUserEventLog, 0, 0) return newOLAPRepository(shared, olapRetentionPeriod, shouldPartitionEventsTables, shouldPartitionOtelTables, statusUpdateBatchSizeLimits), cleanupShared } diff --git a/pkg/repository/olap_status_update_test.go b/pkg/repository/olap_status_update_test.go index 5ce0054ca7..9d2a95914e 100644 --- a/pkg/repository/olap_status_update_test.go +++ b/pkg/repository/olap_status_update_test.go @@ -69,6 +69,9 @@ func createOLAPRepositoryWithPayloadStore(t *testing.T, pool *pgxpool.Pool) *OLA limits.LimitConfigFile{}, false, time.Minute, + false, + 0, + 0, ) t.Cleanup(func() { _ = cleanupShared() }) diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index c5947d8f5f..dfffa4ae90 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -104,10 +104,13 @@ func NewRepository( statusUpdateBatchSizeLimits StatusUpdateBatchSizeLimits, tenantLimitConfig limits.LimitConfigFile, enforceLimits bool, + enableDurableUserEventLog bool, + idempotencyKeyTTL time.Duration, + idempotencyKeyDenyRecheckInterval time.Duration, ) (Repository, func() error) { v := validator.NewDefaultValidator() - shared, cleanupShared := newSharedRepository(pool, ddlPool, v, l, payloadStoreOpts, tenantLimitConfig, enforceLimits, cacheDuration) + shared, cleanupShared := newSharedRepository(pool, ddlPool, v, l, payloadStoreOpts, tenantLimitConfig, enforceLimits, cacheDuration, enableDurableUserEventLog, idempotencyKeyTTL, idempotencyKeyDenyRecheckInterval) mq, cleanupMq := newMessageQueueRepository(shared) diff --git a/pkg/repository/scheduler_optimistic.go b/pkg/repository/scheduler_optimistic.go index c26796ff3e..7bd81b0e7c 100644 --- a/pkg/repository/scheduler_optimistic.go +++ b/pkg/repository/scheduler_optimistic.go @@ -73,12 +73,46 @@ func (r *optimisticSchedulingRepositoryImpl) TriggerFromEvents(ctx context.Conte } func (r *optimisticSchedulingRepositoryImpl) TriggerFromNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*sqlcv1.V1QueueItem, []*V1TaskWithPayload, []*DAGWithData, error) { - triggerOpts, err := r.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) + triggerOpts, denyUpdateKeys, duplicateKeys, err := r.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) if err != nil { + if errors.Is(err, ErrIdempotencyKeyAlreadyClaimed) && len(denyUpdateKeys) > 0 { + tx.Rollback() + updateErr := r.queries.UpdateIdempotencyKeysLastDeniedAt(ctx, r.pool, sqlcv1.UpdateIdempotencyKeysLastDeniedAtParams{ + Tenantid: tenantId, + Keys: denyUpdateKeys, + }) + if updateErr != nil { + err = errors.Join(err, fmt.Errorf("failed to update idempotency key deny timestamps: %w", updateErr)) + } + } + return nil, nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) } + if len(duplicateKeys) > 0 && len(triggerOpts) > 0 { + r.l.Warn(). + Str("tenantId", tenantId.String()). + Int("duplicateKeyCount", len(duplicateKeys)). + Msg("partial idempotency duplicates skipped during optimistic trigger") + } + + if len(denyUpdateKeys) > 0 { + tx.AddPostCommit(func() { + updateErr := r.queries.UpdateIdempotencyKeysLastDeniedAt(ctx, r.pool, sqlcv1.UpdateIdempotencyKeysLastDeniedAtParams{ + Tenantid: tenantId, + Keys: denyUpdateKeys, + }) + if updateErr != nil { + r.l.Error(). + Err(updateErr). + Str("tenantId", tenantId.String()). + Int("keyCount", len(denyUpdateKeys)). + Msg("failed to update idempotency key deny timestamps after optimistic trigger") + } + }) + } + tasks, dags, err := r.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) if err != nil { diff --git a/pkg/repository/shared.go b/pkg/repository/shared.go index 7636640c07..fca943a1fa 100644 --- a/pkg/repository/shared.go +++ b/pkg/repository/shared.go @@ -41,12 +41,15 @@ type sharedRepository struct { stepIdLabelsCache *expirable.LRU[uuid.UUID, []*sqlcv1.GetDesiredLabelsRow] stepIdSlotRequestsCache *expirable.LRU[uuid.UUID, map[string]int32] - celParser *cel.CELParser - env *celgo.Env - celProgramCache *lru.Cache[uint64, celgo.Program] - taskLookupCache *lru.Cache[taskExternalIdTenantIdTuple, *sqlcv1.FlattenExternalIdsRow] - payloadStore PayloadStoreRepository - m TenantLimitRepository + celParser *cel.CELParser + env *celgo.Env + celProgramCache *lru.Cache[uint64, celgo.Program] + taskLookupCache *lru.Cache[taskExternalIdTenantIdTuple, *sqlcv1.FlattenExternalIdsRow] + payloadStore PayloadStoreRepository + m TenantLimitRepository + enableDurableUserEventLog bool + idempotencyKeyTTL time.Duration + idempotencyKeyDenyRecheckInterval time.Duration } func newSharedRepository( @@ -57,6 +60,9 @@ func newSharedRepository( c limits.LimitConfigFile, shouldEnforceLimits bool, cacheDuration time.Duration, + enableDurableUserEventLog bool, + idempotencyKeyTTL time.Duration, + idempotencyKeyDenyRecheckInterval time.Duration, ) (*sharedRepository, func() error) { queries := sqlcv1.New() queueCache := cache.New(5 * time.Minute) @@ -94,23 +100,26 @@ func newSharedRepository( } s := &sharedRepository{ - pool: pool, - ddlPool: ddlPool, - v: v, - l: l, - queries: queries, - queueCache: queueCache, - stepExpressionCache: stepExpressionCache, - concurrencyStrategyCache: concurrencyStrategyCache, - tenantIdWorkflowNameCache: tenantIdWorkflowNameCache, - stepsInWorkflowVersionCache: stepsInWorkflowVersionCache, - stepIdLabelsCache: stepIdLabelsCache, - stepIdSlotRequestsCache: stepIdSlotRequestsCache, - celParser: celParser, - env: env, - celProgramCache: celProgramCache, - taskLookupCache: lookupCache, - payloadStore: payloadStore, + pool: pool, + ddlPool: ddlPool, + v: v, + l: l, + queries: queries, + queueCache: queueCache, + stepExpressionCache: stepExpressionCache, + concurrencyStrategyCache: concurrencyStrategyCache, + tenantIdWorkflowNameCache: tenantIdWorkflowNameCache, + stepsInWorkflowVersionCache: stepsInWorkflowVersionCache, + stepIdLabelsCache: stepIdLabelsCache, + stepIdSlotRequestsCache: stepIdSlotRequestsCache, + celParser: celParser, + env: env, + celProgramCache: celProgramCache, + taskLookupCache: lookupCache, + payloadStore: payloadStore, + enableDurableUserEventLog: enableDurableUserEventLog, + idempotencyKeyTTL: idempotencyKeyTTL, + idempotencyKeyDenyRecheckInterval: idempotencyKeyDenyRecheckInterval, } tenantLimitRepository := newTenantLimitRepository(s, c, shouldEnforceLimits, cacheDuration) diff --git a/pkg/repository/sqlcv1/idempotency-keys.sql b/pkg/repository/sqlcv1/idempotency-keys.sql index d86b710abd..10966e2c7f 100644 --- a/pkg/repository/sqlcv1/idempotency-keys.sql +++ b/pkg/repository/sqlcv1/idempotency-keys.sql @@ -8,7 +8,33 @@ VALUES ( @tenantId::UUID, @key::TEXT, @expiresAt::TIMESTAMPTZ -); +) +ON CONFLICT (tenant_id, key) DO UPDATE +SET + expires_at = EXCLUDED.expires_at, + updated_at = NOW() +WHERE + v1_idempotency_key.claimed_by_external_id IS NULL + AND v1_idempotency_key.expires_at < NOW(); + +-- name: CreateIdempotencyKeys :exec +INSERT INTO v1_idempotency_key ( + tenant_id, + key, + expires_at +) +SELECT + @tenantId::UUID, + k, + @expiresAt::TIMESTAMPTZ +FROM UNNEST(@keys::TEXT[]) AS k +ON CONFLICT (tenant_id, key) DO UPDATE +SET + expires_at = EXCLUDED.expires_at, + updated_at = NOW() +WHERE + v1_idempotency_key.claimed_by_external_id IS NULL + AND v1_idempotency_key.expires_at < NOW(); -- name: CleanUpExpiredIdempotencyKeys :exec DELETE FROM v1_idempotency_key @@ -17,6 +43,59 @@ WHERE AND expires_at < NOW() ; +-- name: DeleteIdempotencyKeysByExternalId :exec +DELETE FROM v1_idempotency_key +WHERE + tenant_id = @tenantId::UUID + AND claimed_by_external_id = @externalId::UUID +; + +-- name: ListIdempotencyKeysByKeys :many +SELECT + tenant_id, + key, + expires_at, + claimed_by_external_id, + last_denied_at, + inserted_at, + updated_at +FROM v1_idempotency_key +WHERE + tenant_id = @tenantId::UUID + AND key = ANY(@keys::TEXT[]) +; + +-- name: UpdateIdempotencyKeysLastDeniedAt :exec +UPDATE v1_idempotency_key +SET + last_denied_at = NOW(), + updated_at = NOW() +WHERE + tenant_id = @tenantId::UUID + AND key = ANY(@keys::TEXT[]) +; + +-- name: UpdateIdempotencyKeysLastDeniedAtSkipLocked :exec +WITH target AS ( + SELECT + tenant_id, + key + FROM v1_idempotency_key + WHERE + tenant_id = @tenantId::UUID + AND key = ANY(@keys::TEXT[]) + FOR UPDATE SKIP LOCKED +) +UPDATE v1_idempotency_key k +SET + last_denied_at = NOW(), + updated_at = NOW() +FROM target t +WHERE + k.tenant_id = t.tenant_id + AND k.key = t.key +; + -- name: ClaimIdempotencyKeys :many WITH inputs AS ( SELECT DISTINCT diff --git a/pkg/repository/sqlcv1/idempotency-keys.sql.go b/pkg/repository/sqlcv1/idempotency-keys.sql.go index 1ef63348e1..e9f4ea36c9 100644 --- a/pkg/repository/sqlcv1/idempotency-keys.sql.go +++ b/pkg/repository/sqlcv1/idempotency-keys.sql.go @@ -55,7 +55,7 @@ WITH inputs AS ( updated_at = NOW() FROM to_update u WHERE (u.tenant_id, u.expires_at, u.key) = (k.tenant_id, k.expires_at, k.key) - RETURNING k.tenant_id, k.key, k.expires_at, k.claimed_by_external_id, k.inserted_at, k.updated_at + RETURNING k.tenant_id, k.key, k.expires_at, k.claimed_by_external_id, k.last_denied_at, k.inserted_at, k.updated_at ) SELECT @@ -128,6 +128,13 @@ VALUES ( $2::TEXT, $3::TIMESTAMPTZ ) +ON CONFLICT (tenant_id, key) DO UPDATE +SET + expires_at = EXCLUDED.expires_at, + updated_at = NOW() +WHERE + v1_idempotency_key.claimed_by_external_id IS NULL + AND v1_idempotency_key.expires_at < NOW() ` type CreateIdempotencyKeyParams struct { @@ -140,3 +147,150 @@ func (q *Queries) CreateIdempotencyKey(ctx context.Context, db DBTX, arg CreateI _, err := db.Exec(ctx, createIdempotencyKey, arg.Tenantid, arg.Key, arg.Expiresat) return err } + +const createIdempotencyKeys = `-- name: CreateIdempotencyKeys :exec +INSERT INTO v1_idempotency_key ( + tenant_id, + key, + expires_at +) +SELECT + $1::UUID, + k, + $2::TIMESTAMPTZ +FROM UNNEST($3::TEXT[]) AS k +ON CONFLICT (tenant_id, key) DO UPDATE +SET + expires_at = EXCLUDED.expires_at, + updated_at = NOW() +WHERE + v1_idempotency_key.claimed_by_external_id IS NULL + AND v1_idempotency_key.expires_at < NOW() +` + +type CreateIdempotencyKeysParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Expiresat pgtype.Timestamptz `json:"expiresat"` + Keys []string `json:"keys"` +} + +func (q *Queries) CreateIdempotencyKeys(ctx context.Context, db DBTX, arg CreateIdempotencyKeysParams) error { + _, err := db.Exec(ctx, createIdempotencyKeys, arg.Tenantid, arg.Expiresat, arg.Keys) + return err +} + +const deleteIdempotencyKeysByExternalId = `-- name: DeleteIdempotencyKeysByExternalId :exec +DELETE FROM v1_idempotency_key +WHERE + tenant_id = $1::UUID + AND claimed_by_external_id = $2::UUID +` + +type DeleteIdempotencyKeysByExternalIdParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Externalid uuid.UUID `json:"externalid"` +} + +func (q *Queries) DeleteIdempotencyKeysByExternalId(ctx context.Context, db DBTX, arg DeleteIdempotencyKeysByExternalIdParams) error { + _, err := db.Exec(ctx, deleteIdempotencyKeysByExternalId, arg.Tenantid, arg.Externalid) + return err +} + +const listIdempotencyKeysByKeys = `-- name: ListIdempotencyKeysByKeys :many +SELECT + tenant_id, + key, + expires_at, + claimed_by_external_id, + last_denied_at, + inserted_at, + updated_at +FROM v1_idempotency_key +WHERE + tenant_id = $1::UUID + AND key = ANY($2::TEXT[]) +` + +type ListIdempotencyKeysByKeysParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Keys []string `json:"keys"` +} + +func (q *Queries) ListIdempotencyKeysByKeys(ctx context.Context, db DBTX, arg ListIdempotencyKeysByKeysParams) ([]*V1IdempotencyKey, error) { + rows, err := db.Query(ctx, listIdempotencyKeysByKeys, arg.Tenantid, arg.Keys) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*V1IdempotencyKey + for rows.Next() { + var i V1IdempotencyKey + if err := rows.Scan( + &i.TenantID, + &i.Key, + &i.ExpiresAt, + &i.ClaimedByExternalID, + &i.LastDeniedAt, + &i.InsertedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateIdempotencyKeysLastDeniedAt = `-- name: UpdateIdempotencyKeysLastDeniedAt :exec +UPDATE v1_idempotency_key +SET + last_denied_at = NOW(), + updated_at = NOW() +WHERE + tenant_id = $1::UUID + AND key = ANY($2::TEXT[]) +` + +type UpdateIdempotencyKeysLastDeniedAtParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Keys []string `json:"keys"` +} + +func (q *Queries) UpdateIdempotencyKeysLastDeniedAt(ctx context.Context, db DBTX, arg UpdateIdempotencyKeysLastDeniedAtParams) error { + _, err := db.Exec(ctx, updateIdempotencyKeysLastDeniedAt, arg.Tenantid, arg.Keys) + return err +} + +const updateIdempotencyKeysLastDeniedAtSkipLocked = `-- name: UpdateIdempotencyKeysLastDeniedAtSkipLocked :exec +WITH target AS ( + SELECT + tenant_id, + key + FROM v1_idempotency_key + WHERE + tenant_id = $1::UUID + AND key = ANY($2::TEXT[]) + FOR UPDATE SKIP LOCKED +) +UPDATE v1_idempotency_key k +SET + last_denied_at = NOW(), + updated_at = NOW() +FROM target t +WHERE + k.tenant_id = t.tenant_id + AND k.key = t.key +` + +type UpdateIdempotencyKeysLastDeniedAtSkipLockedParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Keys []string `json:"keys"` +} + +func (q *Queries) UpdateIdempotencyKeysLastDeniedAtSkipLocked(ctx context.Context, db DBTX, arg UpdateIdempotencyKeysLastDeniedAtSkipLockedParams) error { + _, err := db.Exec(ctx, updateIdempotencyKeysLastDeniedAtSkipLocked, arg.Tenantid, arg.Keys) + return err +} diff --git a/pkg/repository/sqlcv1/models.go b/pkg/repository/sqlcv1/models.go index 016059f636..897eec22ee 100644 --- a/pkg/repository/sqlcv1/models.go +++ b/pkg/repository/sqlcv1/models.go @@ -3283,6 +3283,7 @@ type V1IdempotencyKey struct { Key string `json:"key"` ExpiresAt pgtype.Timestamptz `json:"expires_at"` ClaimedByExternalID *uuid.UUID `json:"claimed_by_external_id"` + LastDeniedAt pgtype.Timestamptz `json:"last_denied_at"` InsertedAt pgtype.Timestamptz `json:"inserted_at"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` } diff --git a/pkg/repository/sqlcv1/olap.sql b/pkg/repository/sqlcv1/olap.sql index 7b64bcff28..f806ceb231 100644 --- a/pkg/repository/sqlcv1/olap.sql +++ b/pkg/repository/sqlcv1/olap.sql @@ -1648,6 +1648,41 @@ LEFT JOIN error_message e ON true LEFT JOIN output_event_external_id o ON true ORDER BY r.inserted_at DESC; +-- name: ReadWorkflowRunStatusByExternalId :one +SELECT + r.tenant_id, + r.external_id, + r.readable_status +FROM + v1_lookup_table_olap lt +JOIN + v1_runs_olap r ON r.inserted_at = lt.inserted_at + AND ( + (lt.dag_id IS NOT NULL AND r.id = lt.dag_id) + OR (lt.task_id IS NOT NULL AND r.id = lt.task_id) + ) +WHERE + lt.external_id = @workflowRunExternalId::uuid +; + +-- name: ReadWorkflowRunStatusesByExternalIds :many +SELECT + r.tenant_id, + r.external_id, + r.readable_status +FROM + v1_lookup_table_olap lt +JOIN + v1_runs_olap r ON r.inserted_at = lt.inserted_at + AND ( + (lt.dag_id IS NOT NULL AND r.id = lt.dag_id) + OR (lt.task_id IS NOT NULL AND r.id = lt.task_id) + ) +WHERE + lt.external_id = ANY(@workflowRunExternalIds::uuid[]) + AND lt.tenant_id = @tenantId::uuid +; + -- name: GetWorkflowRunIdFromDagIdInsertedAt :one SELECT external_id FROM v1_dags_olap diff --git a/pkg/repository/sqlcv1/olap.sql.go b/pkg/repository/sqlcv1/olap.sql.go index d772d28d90..66185c543f 100644 --- a/pkg/repository/sqlcv1/olap.sql.go +++ b/pkg/repository/sqlcv1/olap.sql.go @@ -3302,6 +3302,85 @@ func (q *Queries) ReadWorkflowRunByExternalId(ctx context.Context, db DBTX, work return &i, err } +const readWorkflowRunStatusByExternalId = `-- name: ReadWorkflowRunStatusByExternalId :one +SELECT + r.tenant_id, + r.external_id, + r.readable_status +FROM + v1_lookup_table_olap lt +JOIN + v1_runs_olap r ON r.inserted_at = lt.inserted_at + AND ( + (lt.dag_id IS NOT NULL AND r.id = lt.dag_id) + OR (lt.task_id IS NOT NULL AND r.id = lt.task_id) + ) +WHERE + lt.external_id = $1::uuid +` + +type ReadWorkflowRunStatusByExternalIdRow struct { + TenantID uuid.UUID `json:"tenant_id"` + ExternalID uuid.UUID `json:"external_id"` + ReadableStatus V1ReadableStatusOlap `json:"readable_status"` +} + +func (q *Queries) ReadWorkflowRunStatusByExternalId(ctx context.Context, db DBTX, workflowrunexternalid uuid.UUID) (*ReadWorkflowRunStatusByExternalIdRow, error) { + row := db.QueryRow(ctx, readWorkflowRunStatusByExternalId, workflowrunexternalid) + var i ReadWorkflowRunStatusByExternalIdRow + err := row.Scan(&i.TenantID, &i.ExternalID, &i.ReadableStatus) + return &i, err +} + +const readWorkflowRunStatusesByExternalIds = `-- name: ReadWorkflowRunStatusesByExternalIds :many +SELECT + r.tenant_id, + r.external_id, + r.readable_status +FROM + v1_lookup_table_olap lt +JOIN + v1_runs_olap r ON r.inserted_at = lt.inserted_at + AND ( + (lt.dag_id IS NOT NULL AND r.id = lt.dag_id) + OR (lt.task_id IS NOT NULL AND r.id = lt.task_id) + ) +WHERE + lt.external_id = ANY($1::uuid[]) + AND lt.tenant_id = $2::uuid +` + +type ReadWorkflowRunStatusesByExternalIdsParams struct { + Workflowrunexternalids []uuid.UUID `json:"workflowrunexternalids"` + Tenantid uuid.UUID `json:"tenantid"` +} + +type ReadWorkflowRunStatusesByExternalIdsRow struct { + TenantID uuid.UUID `json:"tenant_id"` + ExternalID uuid.UUID `json:"external_id"` + ReadableStatus V1ReadableStatusOlap `json:"readable_status"` +} + +func (q *Queries) ReadWorkflowRunStatusesByExternalIds(ctx context.Context, db DBTX, arg ReadWorkflowRunStatusesByExternalIdsParams) ([]*ReadWorkflowRunStatusesByExternalIdsRow, error) { + rows, err := db.Query(ctx, readWorkflowRunStatusesByExternalIds, arg.Workflowrunexternalids, arg.Tenantid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*ReadWorkflowRunStatusesByExternalIdsRow + for rows.Next() { + var i ReadWorkflowRunStatusesByExternalIdsRow + if err := rows.Scan(&i.TenantID, &i.ExternalID, &i.ReadableStatus); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const reconcileTaskStatusesFromEvents = `-- name: ReconcileTaskStatusesFromEvents :many WITH inputs AS ( SELECT diff --git a/pkg/repository/sqlcv1/tasks.sql b/pkg/repository/sqlcv1/tasks.sql index 163292ee09..0cbe37ad21 100644 --- a/pkg/repository/sqlcv1/tasks.sql +++ b/pkg/repository/sqlcv1/tasks.sql @@ -906,6 +906,63 @@ WHERE t.tenant_id = @tenantId::uuid AND dt.dag_id = ANY(@dagIds::bigint[]); +-- name: ReadWorkflowRunTerminalStatesByExternalIds :many +WITH input AS ( + SELECT + UNNEST(@externalIds::uuid[]) AS external_id +), lookup AS ( + SELECT + lt.external_id, + lt.dag_id, + lt.task_id, + lt.inserted_at + FROM v1_lookup_table lt + JOIN input i ON i.external_id = lt.external_id + WHERE + lt.tenant_id = @tenantId::uuid +), dag_tasks AS ( + SELECT + l.external_id, + t.id, + t.inserted_at, + t.retry_count + FROM lookup l + JOIN v1_dag_to_task dt ON dt.dag_id = l.dag_id AND dt.dag_inserted_at = l.inserted_at + JOIN v1_task t ON t.id = dt.task_id AND t.inserted_at = dt.task_inserted_at + WHERE l.dag_id IS NOT NULL +), task_only AS ( + SELECT + l.external_id, + t.id, + t.inserted_at, + t.retry_count + FROM lookup l + JOIN v1_task t ON t.id = l.task_id AND t.inserted_at = l.inserted_at + WHERE l.task_id IS NOT NULL +), all_tasks AS ( + SELECT * FROM dag_tasks + UNION ALL + SELECT * FROM task_only +), terminal_tasks AS ( + SELECT + at.external_id, + EXISTS ( + SELECT 1 + FROM v1_task_event e + WHERE + e.tenant_id = @tenantId::uuid + AND (e.task_id, e.task_inserted_at, e.retry_count) = (at.id, at.inserted_at, at.retry_count) + AND e.event_type = ANY('{COMPLETED, FAILED, CANCELLED}'::v1_task_event_type[]) + ) AS is_terminal + FROM all_tasks at +) +SELECT + external_id, + BOOL_AND(is_terminal) AS all_terminal, + COUNT(*) AS task_count +FROM terminal_tasks +GROUP BY external_id; + -- name: ListTaskExpressionEvals :many WITH input AS ( SELECT diff --git a/pkg/repository/sqlcv1/tasks.sql.go b/pkg/repository/sqlcv1/tasks.sql.go index 0bff210b4f..d52d205f5d 100644 --- a/pkg/repository/sqlcv1/tasks.sql.go +++ b/pkg/repository/sqlcv1/tasks.sql.go @@ -2655,6 +2655,95 @@ func (q *Queries) ProcessRetryQueueItems(ctx context.Context, db DBTX, arg Proce return items, nil } +const readWorkflowRunTerminalStatesByExternalIds = `-- name: ReadWorkflowRunTerminalStatesByExternalIds :many +WITH input AS ( + SELECT + UNNEST($1::uuid[]) AS external_id +), lookup AS ( + SELECT + lt.external_id, + lt.dag_id, + lt.task_id, + lt.inserted_at + FROM v1_lookup_table lt + JOIN input i ON i.external_id = lt.external_id + WHERE + lt.tenant_id = $2::uuid +), dag_tasks AS ( + SELECT + l.external_id, + t.id, + t.inserted_at, + t.retry_count + FROM lookup l + JOIN v1_dag_to_task dt ON dt.dag_id = l.dag_id AND dt.dag_inserted_at = l.inserted_at + JOIN v1_task t ON t.id = dt.task_id AND t.inserted_at = dt.task_inserted_at + WHERE l.dag_id IS NOT NULL +), task_only AS ( + SELECT + l.external_id, + t.id, + t.inserted_at, + t.retry_count + FROM lookup l + JOIN v1_task t ON t.id = l.task_id AND t.inserted_at = l.inserted_at + WHERE l.task_id IS NOT NULL +), all_tasks AS ( + SELECT external_id, id, inserted_at, retry_count FROM dag_tasks + UNION ALL + SELECT external_id, id, inserted_at, retry_count FROM task_only +), terminal_tasks AS ( + SELECT + at.external_id, + EXISTS ( + SELECT 1 + FROM v1_task_event e + WHERE + e.tenant_id = $2::uuid + AND (e.task_id, e.task_inserted_at, e.retry_count) = (at.id, at.inserted_at, at.retry_count) + AND e.event_type = ANY('{COMPLETED, FAILED, CANCELLED}'::v1_task_event_type[]) + ) AS is_terminal + FROM all_tasks at +) +SELECT + external_id, + BOOL_AND(is_terminal) AS all_terminal, + COUNT(*) AS task_count +FROM terminal_tasks +GROUP BY external_id +` + +type ReadWorkflowRunTerminalStatesByExternalIdsParams struct { + Externalids []uuid.UUID `json:"externalids"` + Tenantid uuid.UUID `json:"tenantid"` +} + +type ReadWorkflowRunTerminalStatesByExternalIdsRow struct { + ExternalID uuid.UUID `json:"external_id"` + AllTerminal bool `json:"all_terminal"` + TaskCount int64 `json:"task_count"` +} + +func (q *Queries) ReadWorkflowRunTerminalStatesByExternalIds(ctx context.Context, db DBTX, arg ReadWorkflowRunTerminalStatesByExternalIdsParams) ([]*ReadWorkflowRunTerminalStatesByExternalIdsRow, error) { + rows, err := db.Query(ctx, readWorkflowRunTerminalStatesByExternalIds, arg.Externalids, arg.Tenantid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*ReadWorkflowRunTerminalStatesByExternalIdsRow + for rows.Next() { + var i ReadWorkflowRunTerminalStatesByExternalIdsRow + if err := rows.Scan(&i.ExternalID, &i.AllTerminal, &i.TaskCount); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const refreshTimeoutBy = `-- name: RefreshTimeoutBy :one WITH task AS ( SELECT diff --git a/pkg/repository/task_partition_test.go b/pkg/repository/task_partition_test.go index e3f8039626..3ef139bbb4 100644 --- a/pkg/repository/task_partition_test.go +++ b/pkg/repository/task_partition_test.go @@ -4,13 +4,16 @@ package repository import ( "context" + "fmt" "os" + "strings" "sync" "sync/atomic" "testing" "time" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -24,6 +27,36 @@ import ( ) func setupPostgresWithMigration(t *testing.T) (*pgxpool.Pool, func()) { + t.Helper() + + testPoolOnce.Do(func() { + testPoolErr = initTestPool(t) + }) + + require.NoError(t, testPoolErr) + + testPoolMu.Lock() + t.Cleanup(func() { + testPoolMu.Unlock() + }) + + err := resetDatabase(context.Background(), testPool) + require.NoError(t, err) + + return testPool, func() {} +} + +var ( + testPoolOnce sync.Once + testPoolMu sync.Mutex + testPool *pgxpool.Pool + testPoolErr error + + testContainer *postgres.PostgresContainer + originalDatabaseURL string +) + +func initTestPool(t *testing.T) error { ctx := context.Background() postgresContainer, err := postgres.Run(ctx, @@ -37,23 +70,30 @@ func setupPostgresWithMigration(t *testing.T) (*pgxpool.Pool, func()) { WithStartupTimeout(30*time.Second), ), ) - require.NoError(t, err) + if err != nil { + return err + } connStr, err := postgresContainer.ConnectionString(ctx, "sslmode=disable") - require.NoError(t, err) + if err != nil { + return err + } t.Logf("PostgreSQL container started with connection string: %s", connStr) - originalDatabaseURL := os.Getenv("DATABASE_URL") - err = os.Setenv("DATABASE_URL", connStr) - require.NoError(t, err) + originalDatabaseURL = os.Getenv("DATABASE_URL") + if err := os.Setenv("DATABASE_URL", connStr); err != nil { + return err + } t.Log("Running database migration...") migrate.RunMigrations(ctx) t.Log("Migration completed successfully") config, err := pgxpool.ParseConfig(connStr) - require.NoError(t, err) + if err != nil { + return err + } // Set timezone to UTC for all test connections config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { @@ -62,22 +102,107 @@ func setupPostgresWithMigration(t *testing.T) (*pgxpool.Pool, func()) { } pool, err := pgxpool.NewWithConfig(ctx, config) - require.NoError(t, err) + if err != nil { + return err + } - err = pool.Ping(ctx) - require.NoError(t, err) + if err := pool.Ping(ctx); err != nil { + return err + } - cleanup := func() { - pool.Close() - postgresContainer.Terminate(ctx) - if originalDatabaseURL != "" { - os.Setenv("DATABASE_URL", originalDatabaseURL) - } else { - os.Unsetenv("DATABASE_URL") + testPool = pool + testContainer = postgresContainer + + return nil +} + +func resetDatabase(ctx context.Context, pool *pgxpool.Pool) error { + if err := dropPartitionTables(ctx, pool); err != nil { + return err + } + + rows, err := pool.Query(ctx, ` + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + `) + if err != nil { + return err + } + defer rows.Close() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return err } + if table == "goose_db_version" { + continue + } + tables = append(tables, pgx.Identifier{table}.Sanitize()) + } + if rows.Err() != nil { + return rows.Err() + } + + if len(tables) == 0 { + return nil + } + + stmt := fmt.Sprintf("TRUNCATE TABLE %s RESTART IDENTITY CASCADE", strings.Join(tables, ", ")) + if _, err = pool.Exec(ctx, stmt); err != nil { + return err + } + + queries := sqlcv1.New() + today := time.Now().UTC() + return queries.CreatePartitions(ctx, pool, pgtype.Date{ + Time: today, + Valid: true, + }) +} + +func dropPartitionTables(ctx context.Context, pool *pgxpool.Pool) error { + rows, err := pool.Query(ctx, ` + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND ( + tablename ~ '^v1_task_[0-9]{8}$' + OR tablename ~ '^v1_dag_[0-9]{8}$' + OR tablename ~ '^v1_task_event_[0-9]{8}$' + OR tablename ~ '^v1_log_line_[0-9]{8}$' + OR tablename ~ '^v1_payload_[0-9]{8}$' + OR tablename ~ '^v1_event_[0-9]{8}$' + OR tablename ~ '^v1_event_lookup_table_[0-9]{8}$' + OR tablename ~ '^v1_event_to_run_[0-9]{8}$' + ) + `) + if err != nil { + return err + } + defer rows.Close() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return err + } + tables = append(tables, pgx.Identifier{table}.Sanitize()) + } + if rows.Err() != nil { + return rows.Err() + } + + if len(tables) == 0 { + return nil } - return pool, cleanup + stmt := fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", strings.Join(tables, ", ")) + _, err = pool.Exec(ctx, stmt) + return err } func createTaskRepository(pool *pgxpool.Pool) *TaskRepositoryImpl { diff --git a/pkg/repository/trigger.go b/pkg/repository/trigger.go index 5f33141438..b40766c9d1 100644 --- a/pkg/repository/trigger.go +++ b/pkg/repository/trigger.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -372,14 +373,26 @@ func getEventExternalIdToRuns(opts []EventTriggerOpts, externalIdToEventIdAndFil return eventExternalIdToRuns } -func (s *sharedRepository) triggerFromWorkflowNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, error) { - triggerOpts, err := s.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) +func (s *sharedRepository) triggerFromWorkflowNames(ctx context.Context, tx *OptimisticTx, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, []string, error) { + triggerOpts, denyUpdateKeys, duplicateKeys, err := s.prepareTriggerFromWorkflowNames(ctx, tx.tx, tenantId, opts) if err != nil { - return nil, nil, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) + return nil, nil, denyUpdateKeys, fmt.Errorf("failed to prepare trigger from workflow names: %w", err) } - return s.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) + if len(duplicateKeys) > 0 && len(triggerOpts) > 0 { + s.l.Warn(). + Str("tenantId", tenantId.String()). + Int("duplicateKeyCount", len(duplicateKeys)). + Msg("partial idempotency duplicates skipped during trigger") + } + + tasks, dags, err := s.triggerWorkflows(ctx, tx, tenantId, triggerOpts, nil) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to trigger workflows: %w", err) + } + + return tasks, dags, denyUpdateKeys, nil } func (r *TriggerRepositoryImpl) TriggerFromWorkflowNames(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ([]*V1TaskWithPayload, []*DAGWithData, error) { @@ -391,9 +404,19 @@ func (r *TriggerRepositoryImpl) TriggerFromWorkflowNames(ctx context.Context, te defer tx.Rollback() - tasks, dags, err := r.triggerFromWorkflowNames(ctx, tx, tenantId, opts) - + tasks, dags, denyUpdateKeys, err := r.triggerFromWorkflowNames(ctx, tx, tenantId, opts) if err != nil { + if errors.Is(err, ErrIdempotencyKeyAlreadyClaimed) && len(denyUpdateKeys) > 0 { + tx.Rollback() + updateErr := r.queries.UpdateIdempotencyKeysLastDeniedAt(ctx, r.pool, sqlcv1.UpdateIdempotencyKeysLastDeniedAtParams{ + Tenantid: tenantId, + Keys: denyUpdateKeys, + }) + if updateErr != nil { + err = errors.Join(err, fmt.Errorf("failed to update idempotency key deny timestamps: %w", updateErr)) + } + } + return nil, nil, err } @@ -401,6 +424,20 @@ func (r *TriggerRepositoryImpl) TriggerFromWorkflowNames(ctx context.Context, te return nil, nil, err } + if len(denyUpdateKeys) > 0 { + updateErr := r.queries.UpdateIdempotencyKeysLastDeniedAt(ctx, r.pool, sqlcv1.UpdateIdempotencyKeysLastDeniedAtParams{ + Tenantid: tenantId, + Keys: denyUpdateKeys, + }) + if updateErr != nil { + r.l.Error(). + Err(updateErr). + Str("tenantId", tenantId.String()). + Int("keyCount", len(denyUpdateKeys)). + Msg("failed to update idempotency key deny timestamps") + } + } + return tasks, dags, nil } @@ -412,6 +449,31 @@ func (e *ErrNamesNotFound) Error() string { return fmt.Sprintf("workflow names not found: %s", strings.Join(e.Names, ", ")) } +var ErrIdempotencyKeyAlreadyClaimed = errors.New("idempotency key already claimed") + +type IdempotencyKeyAlreadyClaimedError struct { + Keys []string +} + +func (e *IdempotencyKeyAlreadyClaimedError) Error() string { + return fmt.Sprintf("idempotency key already claimed: %s", strings.Join(e.Keys, ", ")) +} + +func (e *IdempotencyKeyAlreadyClaimedError) Is(target error) bool { + return target == ErrIdempotencyKeyAlreadyClaimed +} + +func isTerminalReadableStatus(status sqlcv1.V1ReadableStatusOlap) bool { + switch status { + case sqlcv1.V1ReadableStatusOlapCOMPLETED, + sqlcv1.V1ReadableStatusOlapFAILED, + sqlcv1.V1ReadableStatusOlapCANCELLED: + return true + default: + return false + } +} + func (r *TriggerRepositoryImpl) PreflightVerifyWorkflowNameOpts(ctx context.Context, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) error { // get a list of workflow names workflowNamesFound := make(map[string]bool) @@ -2278,12 +2340,17 @@ func (r *sharedRepository) prepareTriggerFromEvents(ctx context.Context, tx sqlc func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, tx sqlcv1.DBTX, tenantId uuid.UUID, opts []*WorkflowNameTriggerOpts) ( []triggerTuple, + []string, + []string, error, ) { workflowNames := make([]string, 0, len(opts)) uniqueNames := make(map[string]struct{}) namesToOpts := make(map[string][]*WorkflowNameTriggerOpts) idempotencyKeyToExternalIds := make(map[IdempotencyKey]uuid.UUID) + var err error + var denyUpdateKeys []string + var duplicateKeys []string for _, opt := range opts { if opt.IdempotencyKey != nil { @@ -2309,20 +2376,77 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, }) } - keyClaimantPairToWasClaimed, err := claimIdempotencyKeys(ctx, r.queries, tx, tenantId, keyClaimantPairs) + keyClaimantPairToWasClaimed := make(map[KeyClaimantPair]WasSuccessfullyClaimed) - if err != nil { - return nil, fmt.Errorf("failed to claim idempotency keys: %w", err) + if len(keyClaimantPairs) > 0 { + keys := make([]string, 0, len(keyClaimantPairs)) + + for _, pair := range keyClaimantPairs { + keys = append(keys, string(pair.IdempotencyKey)) + } + + ttl := r.idempotencyKeyTTL + if ttl <= 0 { + ttl = 24 * time.Hour + } + expiresAt := sqlchelpers.TimestamptzFromTime(time.Now().Add(ttl)) + + err = r.queries.CreateIdempotencyKeys(ctx, tx, sqlcv1.CreateIdempotencyKeysParams{ + Tenantid: tenantId, + Keys: keys, + Expiresat: expiresAt, + }) + + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create idempotency keys: %w", err) + } + + keyClaimantPairToWasClaimed, err = claimIdempotencyKeys(ctx, r.queries, tx, tenantId, keyClaimantPairs) + + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to claim idempotency keys: %w", err) + } + } + + if len(keyClaimantPairs) > 0 { + unclaimedKeys := make([]string, 0) + + for _, pair := range keyClaimantPairs { + if !keyClaimantPairToWasClaimed[pair] { + unclaimedKeys = append(unclaimedKeys, string(pair.IdempotencyKey)) + } + } + + if len(unclaimedKeys) > 0 { + denyUpdateKeys, err = r.tryReclaimIdempotencyKeys(ctx, tx, tenantId, keyClaimantPairs, keyClaimantPairToWasClaimed, unclaimedKeys) + + if err != nil { + return nil, nil, nil, err + } + + unclaimedKeys = unclaimedKeys[:0] + + for _, pair := range keyClaimantPairs { + if !keyClaimantPairToWasClaimed[pair] { + unclaimedKeys = append(unclaimedKeys, string(pair.IdempotencyKey)) + } + } + + if len(unclaimedKeys) > 0 { + duplicateKeys = append(duplicateKeys, unclaimedKeys...) + } + } } workflowVersionsByNames, err := r.listWorkflowsByNames(ctx, tx, tenantId, workflowNames) if err != nil { - return nil, fmt.Errorf("failed to list workflows for names: %w", err) + return nil, nil, nil, fmt.Errorf("failed to list workflows for names: %w", err) } // each (workflowVersionId, opt) is a separate workflow that we need to create triggerOpts := make([]triggerTuple, 0, len(opts)) + allowPartialIdempotency := len(opts) > 0 for _, workflowVersion := range workflowVersionsByNames { opts, ok := namesToOpts[workflowVersion.WorkflowName] @@ -2332,6 +2456,8 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, } for _, opt := range opts { + allowPartialIdempotency = allowPartialIdempotency && opt.AllowPartialIdempotency + if opt.IdempotencyKey != nil { keyClaimantPair := KeyClaimantPair{ IdempotencyKey: *opt.IdempotencyKey, @@ -2365,7 +2491,15 @@ func (r *sharedRepository) prepareTriggerFromWorkflowNames(ctx context.Context, } } - return triggerOpts, nil + if len(triggerOpts) > 0 && len(duplicateKeys) > 0 && !allowPartialIdempotency { + return nil, denyUpdateKeys, duplicateKeys, &IdempotencyKeyAlreadyClaimedError{Keys: duplicateKeys} + } + + if len(triggerOpts) == 0 && len(duplicateKeys) > 0 { + return nil, denyUpdateKeys, duplicateKeys, &IdempotencyKeyAlreadyClaimedError{Keys: duplicateKeys} + } + + return triggerOpts, denyUpdateKeys, duplicateKeys, nil } type TriggerOptInvalidArgumentError struct { @@ -2487,3 +2621,218 @@ func injectParentIDs(additionalMetadata []byte, parentWorkflowRunID, parentStepR } return out } + +func (r *sharedRepository) tryReclaimIdempotencyKeys( + ctx context.Context, + tx sqlcv1.DBTX, + tenantId uuid.UUID, + keyClaimantPairs []KeyClaimantPair, + keyClaimantPairToWasClaimed map[KeyClaimantPair]WasSuccessfullyClaimed, + unclaimedKeys []string, +) ([]string, error) { + if len(unclaimedKeys) == 0 { + return nil, nil + } + + if r.idempotencyKeyDenyRecheckInterval <= 0 { + return nil, nil + } + + rows, err := r.queries.ListIdempotencyKeysByKeys(ctx, r.pool, sqlcv1.ListIdempotencyKeysByKeysParams{ + Tenantid: tenantId, + Keys: unclaimedKeys, + }) + + if err != nil { + return nil, fmt.Errorf("failed to list idempotency keys for recheck: %w", err) + } + + if len(rows) == 0 { + return nil, nil + } + + now := time.Now() + throttledCount := 0 + keysToCheck := make([]string, 0, len(rows)) + keysToReclaim := make([]string, 0) + keysToReclaimSet := make(map[string]struct{}) + keyToExternalId := make(map[string]uuid.UUID) + externalIdsToCheck := make(map[uuid.UUID]struct{}) + + for _, row := range rows { + if row.ClaimedByExternalID == nil { + continue + } + + if row.LastDeniedAt.Valid { + if now.Sub(row.LastDeniedAt.Time) < r.idempotencyKeyDenyRecheckInterval { + throttledCount++ + continue + } + } + + keysToCheck = append(keysToCheck, row.Key) + keyToExternalId[row.Key] = *row.ClaimedByExternalID + externalIdsToCheck[*row.ClaimedByExternalID] = struct{}{} + } + + if len(keysToCheck) == 0 { + if throttledCount > 0 { + r.l.Debug(). + Str("tenantId", tenantId.String()). + Int("throttledCount", throttledCount). + Msg("idempotency recheck throttled") + } + return nil, nil + } + + if throttledCount > 0 { + r.l.Debug(). + Str("tenantId", tenantId.String()). + Int("throttledCount", throttledCount). + Msg("idempotency recheck throttled") + } + + externalIdToStatus := make(map[uuid.UUID]sqlcv1.V1ReadableStatusOlap) + + externalIds := make([]uuid.UUID, 0, len(externalIdsToCheck)) + for externalId := range externalIdsToCheck { + externalIds = append(externalIds, externalId) + } + + statusRows, statusErr := r.queries.ReadWorkflowRunStatusesByExternalIds(ctx, r.pool, sqlcv1.ReadWorkflowRunStatusesByExternalIdsParams{ + Workflowrunexternalids: externalIds, + Tenantid: tenantId, + }) + if statusErr != nil { + r.l.Error(). + Err(statusErr). + Str("tenantId", tenantId.String()). + Int("externalIdCount", len(externalIds)). + Msg("failed to read workflow run status for idempotency recheck") + return nil, fmt.Errorf("failed to read workflow run status for idempotency recheck: %w", statusErr) + } + + for _, row := range statusRows { + externalIdToStatus[row.ExternalID] = row.ReadableStatus + } + + missingExternalIds := make([]uuid.UUID, 0) + for externalId := range externalIdsToCheck { + if _, ok := externalIdToStatus[externalId]; !ok { + missingExternalIds = append(missingExternalIds, externalId) + } + } + + terminalFallback := make(map[uuid.UUID]struct{}) + if len(missingExternalIds) > 0 { + fallbackRows, fallbackErr := r.queries.ReadWorkflowRunTerminalStatesByExternalIds(ctx, r.pool, sqlcv1.ReadWorkflowRunTerminalStatesByExternalIdsParams{ + Externalids: missingExternalIds, + Tenantid: tenantId, + }) + if fallbackErr != nil { + r.l.Error(). + Err(fallbackErr). + Str("tenantId", tenantId.String()). + Int("missingOlapCount", len(missingExternalIds)). + Msg("failed to read workflow run status from core tables for idempotency recheck") + } else { + for _, row := range fallbackRows { + if row.TaskCount > 0 && row.AllTerminal { + terminalFallback[row.ExternalID] = struct{}{} + } + } + } + } + + missingDeniedCount := 0 + + for _, key := range keysToCheck { + externalId, ok := keyToExternalId[key] + if !ok { + continue + } + + status, ok := externalIdToStatus[externalId] + terminal := ok && isTerminalReadableStatus(status) + if !ok { + if _, ok := terminalFallback[externalId]; ok { + terminal = true + } else { + missingDeniedCount++ + } + } + + if terminal { + deleteErr := r.queries.DeleteIdempotencyKeysByExternalId(ctx, tx, sqlcv1.DeleteIdempotencyKeysByExternalIdParams{ + Tenantid: tenantId, + Externalid: externalId, + }) + if deleteErr != nil { + return nil, fmt.Errorf("failed to delete idempotency keys for completed workflow run: %w", deleteErr) + } + + keysToReclaim = append(keysToReclaim, key) + keysToReclaimSet[key] = struct{}{} + } + } + + if missingDeniedCount > 0 { + r.l.Warn(). + Str("tenantId", tenantId.String()). + Int("missingStatusDeniedCount", missingDeniedCount). + Msg("idempotency recheck denied due to missing workflow run status") + } + + keysToUpdate := keysToCheck + if len(keysToReclaimSet) > 0 { + keysToUpdate = make([]string, 0, len(keysToCheck)) + for _, key := range keysToCheck { + if _, ok := keysToReclaimSet[key]; !ok { + keysToUpdate = append(keysToUpdate, key) + } + } + } + + if len(keysToReclaim) == 0 { + return keysToUpdate, nil + } + + ttl := r.idempotencyKeyTTL + if ttl <= 0 { + ttl = 24 * time.Hour + } + + expiresAt := sqlchelpers.TimestamptzFromTime(time.Now().Add(ttl)) + + createErr := r.queries.CreateIdempotencyKeys(ctx, tx, sqlcv1.CreateIdempotencyKeysParams{ + Tenantid: tenantId, + Keys: keysToReclaim, + Expiresat: expiresAt, + }) + if createErr != nil { + return nil, fmt.Errorf("failed to recreate idempotency keys after reclaim: %w", createErr) + } + + pairsToReclaim := make([]KeyClaimantPair, 0, len(keysToReclaim)) + for _, pair := range keyClaimantPairs { + if _, ok := keysToReclaimSet[string(pair.IdempotencyKey)]; ok { + pairsToReclaim = append(pairsToReclaim, pair) + } + } + + if len(pairsToReclaim) == 0 { + return keysToUpdate, nil + } + + claimResults, err := claimIdempotencyKeys(ctx, r.queries, tx, tenantId, pairsToReclaim) + if err != nil { + return nil, fmt.Errorf("failed to reclaim idempotency keys: %w", err) + } + + for pair, claimed := range claimResults { + keyClaimantPairToWasClaimed[pair] = claimed + } + + return keysToUpdate, nil +} diff --git a/sdks/python/hatchet_sdk/clients/admin.py b/sdks/python/hatchet_sdk/clients/admin.py index be15f0f3a5..b9798673de 100644 --- a/sdks/python/hatchet_sdk/clients/admin.py +++ b/sdks/python/hatchet_sdk/clients/admin.py @@ -164,6 +164,7 @@ class TriggerWorkflowRequest(BaseModel): desired_worker_label: ( dict[str, DesiredWorkerLabel] | list[DesiredWorkerLabel] | None ) = None + idempotency_key: str | None = None @field_validator("additional_metadata", mode="before") @classmethod @@ -208,7 +209,7 @@ def _prepare_workflow_request( for key, d in labels_dict.items() } - return trigger_protos.TriggerWorkflowRequest( + request = trigger_protos.TriggerWorkflowRequest( name=workflow_name, input=input, parent_id=_options.parent_id, @@ -220,8 +221,17 @@ def _prepare_workflow_request( priority=_options.priority, desired_worker_labels=desired_worker_labels, ) + if ( + (_options.idempotency_key or options.key) + and "idempotency_key" in request.DESCRIPTOR.fields_by_name + ): + request.idempotency_key = _options.idempotency_key or options.key - def _parse_schedule(self, schedule: datetime) -> timestamp_pb2.Timestamp: + return request + + def _parse_schedule( + self, schedule: datetime | timestamp_pb2.Timestamp + ) -> timestamp_pb2.Timestamp: if isinstance(schedule, datetime): if not schedule.tzinfo: logger.warning( @@ -395,6 +405,7 @@ def _create_workflow_run_request( sticky=options.sticky, key=options.key, desired_worker_label=options.desired_worker_label, + idempotency_key=options.idempotency_key, ) namespace = options.namespace or self.namespace diff --git a/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.py b/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.py index a1e6b14f14..f20a52e323 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.py +++ b/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17v1/shared/trigger.proto\x12\x02v1\"\xe8\x01\n\x13\x44\x65siredWorkerLabels\x12\x16\n\tstr_value\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tint_value\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08required\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x32\n\ncomparator\x18\x04 \x01(\x0e\x32\x19.v1.WorkerLabelComparatorH\x03\x88\x01\x01\x12\x13\n\x06weight\x18\x05 \x01(\x05H\x04\x88\x01\x01\x42\x0c\n\n_str_valueB\x0c\n\n_int_valueB\x0b\n\t_requiredB\r\n\x0b_comparatorB\t\n\x07_weight\"\xb2\x04\n\x16TriggerWorkflowRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\t\x12\x16\n\tparent_id\x18\x03 \x01(\tH\x00\x88\x01\x01\x12(\n\x1bparent_task_run_external_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63hild_index\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tchild_key\x18\x06 \x01(\tH\x03\x88\x01\x01\x12 \n\x13\x61\x64\x64itional_metadata\x18\x07 \x01(\tH\x04\x88\x01\x01\x12\x1e\n\x11\x64\x65sired_worker_id\x18\x08 \x01(\tH\x05\x88\x01\x01\x12\x15\n\x08priority\x18\t \x01(\x05H\x06\x88\x01\x01\x12R\n\x15\x64\x65sired_worker_labels\x18\n \x03(\x0b\x32\x33.v1.TriggerWorkflowRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0c\n\n_parent_idB\x1e\n\x1c_parent_task_run_external_idB\x0e\n\x0c_child_indexB\x0c\n\n_child_keyB\x16\n\x14_additional_metadataB\x14\n\x12_desired_worker_idB\x0b\n\t_priority*\x85\x01\n\x15WorkerLabelComparator\x12\t\n\x05\x45QUAL\x10\x00\x12\r\n\tNOT_EQUAL\x10\x01\x12\x10\n\x0cGREATER_THAN\x10\x02\x12\x19\n\x15GREATER_THAN_OR_EQUAL\x10\x03\x12\r\n\tLESS_THAN\x10\x04\x12\x16\n\x12LESS_THAN_OR_EQUAL\x10\x05\x42\x42Z@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17v1/shared/trigger.proto\x12\x02v1\"\xe8\x01\n\x13\x44\x65siredWorkerLabels\x12\x16\n\tstr_value\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tint_value\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08required\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x32\n\ncomparator\x18\x04 \x01(\x0e\x32\x19.v1.WorkerLabelComparatorH\x03\x88\x01\x01\x12\x13\n\x06weight\x18\x05 \x01(\x05H\x04\x88\x01\x01\x42\x0c\n\n_str_valueB\x0c\n\n_int_valueB\x0b\n\t_requiredB\r\n\x0b_comparatorB\t\n\x07_weight\"\xe4\x04\n\x16TriggerWorkflowRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\t\x12\x16\n\tparent_id\x18\x03 \x01(\tH\x00\x88\x01\x01\x12(\n\x1bparent_task_run_external_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63hild_index\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x16\n\tchild_key\x18\x06 \x01(\tH\x03\x88\x01\x01\x12 \n\x13\x61\x64\x64itional_metadata\x18\x07 \x01(\tH\x04\x88\x01\x01\x12\x1e\n\x11\x64\x65sired_worker_id\x18\x08 \x01(\tH\x05\x88\x01\x01\x12\x15\n\x08priority\x18\t \x01(\x05H\x06\x88\x01\x01\x12R\n\x15\x64\x65sired_worker_labels\x18\n \x03(\x0b\x32\x33.v1.TriggerWorkflowRequest.DesiredWorkerLabelsEntry\x12\x1c\n\x0fidempotency_key\x18\x0b \x01(\tH\x07\x88\x01\x01\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0c\n\n_parent_idB\x1e\n\x1c_parent_task_run_external_idB\x0e\n\x0c_child_indexB\x0c\n\n_child_keyB\x16\n\x14_additional_metadataB\x14\n\x12_desired_worker_idB\x0b\n\t_priorityB\x12\n\x10_idempotency_key*\x85\x01\n\x15WorkerLabelComparator\x12\t\n\x05\x45QUAL\x10\x00\x12\r\n\tNOT_EQUAL\x10\x01\x12\x10\n\x0cGREATER_THAN\x10\x02\x12\x19\n\x15GREATER_THAN_OR_EQUAL\x10\x03\x12\r\n\tLESS_THAN\x10\x04\x12\x16\n\x12LESS_THAN_OR_EQUAL\x10\x05\x42\x42Z@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,12 +34,12 @@ _globals['DESCRIPTOR']._serialized_options = b'Z@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1' _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._loaded_options = None _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._serialized_options = b'8\001' - _globals['_WORKERLABELCOMPARATOR']._serialized_start=832 - _globals['_WORKERLABELCOMPARATOR']._serialized_end=965 + _globals['_WORKERLABELCOMPARATOR']._serialized_start=882 + _globals['_WORKERLABELCOMPARATOR']._serialized_end=1015 _globals['_DESIREDWORKERLABELS']._serialized_start=32 _globals['_DESIREDWORKERLABELS']._serialized_end=264 _globals['_TRIGGERWORKFLOWREQUEST']._serialized_start=267 - _globals['_TRIGGERWORKFLOWREQUEST']._serialized_end=829 - _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._serialized_start=611 - _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._serialized_end=694 + _globals['_TRIGGERWORKFLOWREQUEST']._serialized_end=879 + _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._serialized_start=641 + _globals['_TRIGGERWORKFLOWREQUEST_DESIREDWORKERLABELSENTRY']._serialized_end=724 # @@protoc_insertion_point(module_scope) diff --git a/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.pyi b/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.pyi index 28aefdcb4b..feee61bfda 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.pyi +++ b/sdks/python/hatchet_sdk/contracts/v1/shared/trigger_pb2.pyi @@ -37,7 +37,7 @@ class DesiredWorkerLabels(_message.Message): def __init__(self, str_value: _Optional[str] = ..., int_value: _Optional[int] = ..., required: bool = ..., comparator: _Optional[_Union[WorkerLabelComparator, str]] = ..., weight: _Optional[int] = ...) -> None: ... class TriggerWorkflowRequest(_message.Message): - __slots__ = ("name", "input", "parent_id", "parent_task_run_external_id", "child_index", "child_key", "additional_metadata", "desired_worker_id", "priority", "desired_worker_labels") + __slots__ = ("name", "input", "parent_id", "parent_task_run_external_id", "child_index", "child_key", "additional_metadata", "desired_worker_id", "priority", "desired_worker_labels", "idempotency_key") class DesiredWorkerLabelsEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -55,6 +55,7 @@ class TriggerWorkflowRequest(_message.Message): DESIRED_WORKER_ID_FIELD_NUMBER: _ClassVar[int] PRIORITY_FIELD_NUMBER: _ClassVar[int] DESIRED_WORKER_LABELS_FIELD_NUMBER: _ClassVar[int] + IDEMPOTENCY_KEY_FIELD_NUMBER: _ClassVar[int] name: str input: str parent_id: str @@ -65,4 +66,5 @@ class TriggerWorkflowRequest(_message.Message): desired_worker_id: str priority: int desired_worker_labels: _containers.MessageMap[str, DesiredWorkerLabels] - def __init__(self, name: _Optional[str] = ..., input: _Optional[str] = ..., parent_id: _Optional[str] = ..., parent_task_run_external_id: _Optional[str] = ..., child_index: _Optional[int] = ..., child_key: _Optional[str] = ..., additional_metadata: _Optional[str] = ..., desired_worker_id: _Optional[str] = ..., priority: _Optional[int] = ..., desired_worker_labels: _Optional[_Mapping[str, DesiredWorkerLabels]] = ...) -> None: ... + idempotency_key: str + def __init__(self, name: _Optional[str] = ..., input: _Optional[str] = ..., parent_id: _Optional[str] = ..., parent_task_run_external_id: _Optional[str] = ..., child_index: _Optional[int] = ..., child_key: _Optional[str] = ..., additional_metadata: _Optional[str] = ..., desired_worker_id: _Optional[str] = ..., priority: _Optional[int] = ..., desired_worker_labels: _Optional[_Mapping[str, DesiredWorkerLabels]] = ..., idempotency_key: _Optional[str] = ...) -> None: ... diff --git a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py index 3a9ab133c1..602f2da8e2 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py +++ b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.py @@ -27,7 +27,7 @@ from hatchet_sdk.contracts.v1.shared import trigger_pb2 as v1_dot_shared_dot_trigger__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xae\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priority\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xac\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schema\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12v1/workflows.proto\x12\x02v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19v1/shared/condition.proto\x1a\x17v1/shared/trigger.proto\"[\n\x12\x43\x61ncelTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"[\n\x12ReplayTasksRequest\x12\x14\n\x0c\x65xternal_ids\x18\x01 \x03(\t\x12$\n\x06\x66ilter\x18\x02 \x01(\x0b\x32\x0f.v1.TasksFilterH\x00\x88\x01\x01\x42\t\n\x07_filter\"\xb7\x01\n\x0bTasksFilter\x12\x10\n\x08statuses\x18\x01 \x03(\t\x12)\n\x05since\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\x05until\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12\x14\n\x0cworkflow_ids\x18\x04 \x03(\t\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x03(\tB\x08\n\x06_until\".\n\x13\x43\x61ncelTasksResponse\x12\x17\n\x0f\x63\x61ncelled_tasks\x18\x01 \x03(\t\"-\n\x13ReplayTasksResponse\x12\x16\n\x0ereplayed_tasks\x18\x01 \x03(\t\"\xe0\x02\n\x19TriggerWorkflowRunRequest\x12\x15\n\rworkflow_name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\x0c\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x03 \x01(\x0c\x12\x15\n\x08priority\x18\x04 \x01(\x05H\x00\x88\x01\x01\x12U\n\x15\x64\x65sired_worker_labels\x18\x05 \x03(\x0b\x32\x36.v1.TriggerWorkflowRunRequest.DesiredWorkerLabelsEntry\x12\x1c\n\x0fidempotency_key\x18\x06 \x01(\tH\x01\x88\x01\x01\x1aS\n\x18\x44\x65siredWorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x42\x0b\n\t_priorityB\x12\n\x10_idempotency_key\"1\n\x1aTriggerWorkflowRunResponse\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"X\n\x18\x42ranchDurableTaskRequest\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"Y\n\x19\x42ranchDurableTaskResponse\x12\x18\n\x10task_external_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\x03\x12\x11\n\tbranch_id\x18\x03 \x01(\x03\"\xac\x04\n\x1c\x43reateWorkflowVersionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12!\n\x05tasks\x18\x06 \x03(\x0b\x32\x12.v1.CreateTaskOpts\x12$\n\x0b\x63oncurrency\x18\x07 \x01(\x0b\x32\x0f.v1.Concurrency\x12\x17\n\ncron_input\x18\x08 \x01(\tH\x00\x88\x01\x01\x12\x30\n\x0fon_failure_task\x18\t \x01(\x0b\x32\x12.v1.CreateTaskOptsH\x01\x88\x01\x01\x12\'\n\x06sticky\x18\n \x01(\x0e\x32\x12.v1.StickyStrategyH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65\x66\x61ult_priority\x18\x0b \x01(\x05H\x03\x88\x01\x01\x12(\n\x0f\x63oncurrency_arr\x18\x0c \x03(\x0b\x32\x0f.v1.Concurrency\x12*\n\x0f\x64\x65\x66\x61ult_filters\x18\r \x03(\x0b\x32\x11.v1.DefaultFilter\x12\x1e\n\x11input_json_schema\x18\x0e \x01(\x0cH\x04\x88\x01\x01\x42\r\n\x0b_cron_inputB\x12\n\x10_on_failure_taskB\t\n\x07_stickyB\x13\n\x11_default_priorityB\x14\n\x12_input_json_schema\"T\n\rDefaultFilter\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x14\n\x07payload\x18\x03 \x01(\x0cH\x00\x88\x01\x01\x42\n\n\x08_payload\"\x93\x01\n\x0b\x43oncurrency\x12\x12\n\nexpression\x18\x01 \x01(\t\x12\x15\n\x08max_runs\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x39\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x1c.v1.ConcurrencyLimitStrategyH\x01\x88\x01\x01\x42\x0b\n\t_max_runsB\x11\n\x0f_limit_strategy\"\xb7\x05\n\x0e\x43reateTaskOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x0f\n\x07retries\x18\x06 \x01(\x05\x12,\n\x0brate_limits\x18\x07 \x03(\x0b\x32\x17.v1.CreateTaskRateLimit\x12;\n\rworker_labels\x18\x08 \x03(\x0b\x32$.v1.CreateTaskOpts.WorkerLabelsEntry\x12\x1b\n\x0e\x62\x61\x63koff_factor\x18\t \x01(\x02H\x00\x88\x01\x01\x12 \n\x13\x62\x61\x63koff_max_seconds\x18\n \x01(\x05H\x01\x88\x01\x01\x12$\n\x0b\x63oncurrency\x18\x0b \x03(\x0b\x32\x0f.v1.Concurrency\x12+\n\nconditions\x18\x0c \x01(\x0b\x32\x12.v1.TaskConditionsH\x02\x88\x01\x01\x12\x1d\n\x10schedule_timeout\x18\r \x01(\tH\x03\x88\x01\x01\x12\x12\n\nis_durable\x18\x0e \x01(\x08\x12;\n\rslot_requests\x18\x0f \x03(\x0b\x32$.v1.CreateTaskOpts.SlotRequestsEntry\x1aL\n\x11WorkerLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.v1.DesiredWorkerLabels:\x02\x38\x01\x1a\x33\n\x11SlotRequestsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x11\n\x0f_backoff_factorB\x16\n\x14_backoff_max_secondsB\r\n\x0b_conditionsB\x13\n\x11_schedule_timeout\"\xfd\x01\n\x13\x43reateTaskRateLimit\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\x05units\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08key_expr\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nunits_expr\x18\x04 \x01(\tH\x02\x88\x01\x01\x12\x1e\n\x11limit_values_expr\x18\x05 \x01(\tH\x03\x88\x01\x01\x12,\n\x08\x64uration\x18\x06 \x01(\x0e\x32\x15.v1.RateLimitDurationH\x04\x88\x01\x01\x42\x08\n\x06_unitsB\x0b\n\t_key_exprB\r\n\x0b_units_exprB\x14\n\x12_limit_values_exprB\x0b\n\t_duration\"@\n\x1d\x43reateWorkflowVersionResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\"+\n\x14GetRunDetailsRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"\xaa\x01\n\rTaskRunDetail\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12\x12\n\x05\x65rror\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06output\x18\x04 \x01(\x0cH\x01\x88\x01\x01\x12\x13\n\x0breadable_id\x18\x05 \x01(\t\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x42\x08\n\x06_errorB\t\n\x07_output\"\x84\x02\n\x15GetRunDetailsResponse\x12\r\n\x05input\x18\x01 \x01(\x0c\x12\x1d\n\x06status\x18\x02 \x01(\x0e\x32\r.v1.RunStatus\x12:\n\ttask_runs\x18\x03 \x03(\x0b\x32\'.v1.GetRunDetailsResponse.TaskRunsEntry\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x1b\n\x13\x61\x64\x64itional_metadata\x18\x05 \x01(\x0c\x12\x12\n\nis_evicted\x18\x06 \x01(\x08\x1a\x42\n\rTaskRunsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.v1.TaskRunDetail:\x02\x38\x01*$\n\x0eStickyStrategy\x12\x08\n\x04SOFT\x10\x00\x12\x08\n\x04HARD\x10\x01*]\n\x11RateLimitDuration\x12\n\n\x06SECOND\x10\x00\x12\n\n\x06MINUTE\x10\x01\x12\x08\n\x04HOUR\x10\x02\x12\x07\n\x03\x44\x41Y\x10\x03\x12\x08\n\x04WEEK\x10\x04\x12\t\n\x05MONTH\x10\x05\x12\x08\n\x04YEAR\x10\x06*[\n\tRunStatus\x12\n\n\x06QUEUED\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\r\n\tCANCELLED\x10\x04\x12\x0b\n\x07\x45VICTED\x10\x05*\x7f\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x12\x15\n\x11GROUP_ROUND_ROBIN\x10\x03\x12\x11\n\rCANCEL_NEWEST\x10\x04\x32\xcf\x03\n\x0c\x41\x64minService\x12R\n\x0bPutWorkflow\x12 .v1.CreateWorkflowVersionRequest\x1a!.v1.CreateWorkflowVersionResponse\x12>\n\x0b\x43\x61ncelTasks\x12\x16.v1.CancelTasksRequest\x1a\x17.v1.CancelTasksResponse\x12>\n\x0bReplayTasks\x12\x16.v1.ReplayTasksRequest\x1a\x17.v1.ReplayTasksResponse\x12S\n\x12TriggerWorkflowRun\x12\x1d.v1.TriggerWorkflowRunRequest\x1a\x1e.v1.TriggerWorkflowRunResponse\x12\x44\n\rGetRunDetails\x12\x18.v1.GetRunDetailsRequest\x1a\x19.v1.GetRunDetailsResponse\x12P\n\x11\x42ranchDurableTask\x12\x1c.v1.BranchDurableTaskRequest\x1a\x1d.v1.BranchDurableTaskResponseBBZ@github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -43,14 +43,14 @@ _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_options = b'8\001' _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._loaded_options = None _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_options = b'8\001' - _globals['_STICKYSTRATEGY']._serialized_start=3411 - _globals['_STICKYSTRATEGY']._serialized_end=3447 - _globals['_RATELIMITDURATION']._serialized_start=3449 - _globals['_RATELIMITDURATION']._serialized_end=3542 - _globals['_RUNSTATUS']._serialized_start=3544 - _globals['_RUNSTATUS']._serialized_end=3635 - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=3637 - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=3764 + _globals['_STICKYSTRATEGY']._serialized_start=3461 + _globals['_STICKYSTRATEGY']._serialized_end=3497 + _globals['_RATELIMITDURATION']._serialized_start=3499 + _globals['_RATELIMITDURATION']._serialized_end=3592 + _globals['_RUNSTATUS']._serialized_start=3594 + _globals['_RUNSTATUS']._serialized_end=3685 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=3687 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=3814 _globals['_CANCELTASKSREQUEST']._serialized_start=111 _globals['_CANCELTASKSREQUEST']._serialized_end=202 _globals['_REPLAYTASKSREQUEST']._serialized_start=204 @@ -62,39 +62,39 @@ _globals['_REPLAYTASKSRESPONSE']._serialized_start=531 _globals['_REPLAYTASKSRESPONSE']._serialized_end=576 _globals['_TRIGGERWORKFLOWRUNREQUEST']._serialized_start=579 - _globals['_TRIGGERWORKFLOWRUNREQUEST']._serialized_end=881 - _globals['_TRIGGERWORKFLOWRUNREQUEST_DESIREDWORKERLABELSENTRY']._serialized_start=785 - _globals['_TRIGGERWORKFLOWRUNREQUEST_DESIREDWORKERLABELSENTRY']._serialized_end=868 - _globals['_TRIGGERWORKFLOWRUNRESPONSE']._serialized_start=883 - _globals['_TRIGGERWORKFLOWRUNRESPONSE']._serialized_end=932 - _globals['_BRANCHDURABLETASKREQUEST']._serialized_start=934 - _globals['_BRANCHDURABLETASKREQUEST']._serialized_end=1022 - _globals['_BRANCHDURABLETASKRESPONSE']._serialized_start=1024 - _globals['_BRANCHDURABLETASKRESPONSE']._serialized_end=1113 - _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_start=1116 - _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_end=1672 - _globals['_DEFAULTFILTER']._serialized_start=1674 - _globals['_DEFAULTFILTER']._serialized_end=1758 - _globals['_CONCURRENCY']._serialized_start=1761 - _globals['_CONCURRENCY']._serialized_end=1908 - _globals['_CREATETASKOPTS']._serialized_start=1911 - _globals['_CREATETASKOPTS']._serialized_end=2606 - _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_start=2398 - _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_end=2474 - _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_start=2476 - _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_end=2527 - _globals['_CREATETASKRATELIMIT']._serialized_start=2609 - _globals['_CREATETASKRATELIMIT']._serialized_end=2862 - _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_start=2864 - _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_end=2928 - _globals['_GETRUNDETAILSREQUEST']._serialized_start=2930 - _globals['_GETRUNDETAILSREQUEST']._serialized_end=2973 - _globals['_TASKRUNDETAIL']._serialized_start=2976 - _globals['_TASKRUNDETAIL']._serialized_end=3146 - _globals['_GETRUNDETAILSRESPONSE']._serialized_start=3149 - _globals['_GETRUNDETAILSRESPONSE']._serialized_end=3409 - _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_start=3343 - _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_end=3409 - _globals['_ADMINSERVICE']._serialized_start=3767 - _globals['_ADMINSERVICE']._serialized_end=4230 + _globals['_TRIGGERWORKFLOWRUNREQUEST']._serialized_end=931 + _globals['_TRIGGERWORKFLOWRUNREQUEST_DESIREDWORKERLABELSENTRY']._serialized_start=815 + _globals['_TRIGGERWORKFLOWRUNREQUEST_DESIREDWORKERLABELSENTRY']._serialized_end=898 + _globals['_TRIGGERWORKFLOWRUNRESPONSE']._serialized_start=933 + _globals['_TRIGGERWORKFLOWRUNRESPONSE']._serialized_end=982 + _globals['_BRANCHDURABLETASKREQUEST']._serialized_start=984 + _globals['_BRANCHDURABLETASKREQUEST']._serialized_end=1072 + _globals['_BRANCHDURABLETASKRESPONSE']._serialized_start=1074 + _globals['_BRANCHDURABLETASKRESPONSE']._serialized_end=1163 + _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_start=1166 + _globals['_CREATEWORKFLOWVERSIONREQUEST']._serialized_end=1722 + _globals['_DEFAULTFILTER']._serialized_start=1724 + _globals['_DEFAULTFILTER']._serialized_end=1808 + _globals['_CONCURRENCY']._serialized_start=1811 + _globals['_CONCURRENCY']._serialized_end=1958 + _globals['_CREATETASKOPTS']._serialized_start=1961 + _globals['_CREATETASKOPTS']._serialized_end=2656 + _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_start=2448 + _globals['_CREATETASKOPTS_WORKERLABELSENTRY']._serialized_end=2524 + _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_start=2526 + _globals['_CREATETASKOPTS_SLOTREQUESTSENTRY']._serialized_end=2577 + _globals['_CREATETASKRATELIMIT']._serialized_start=2659 + _globals['_CREATETASKRATELIMIT']._serialized_end=2912 + _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_start=2914 + _globals['_CREATEWORKFLOWVERSIONRESPONSE']._serialized_end=2978 + _globals['_GETRUNDETAILSREQUEST']._serialized_start=2980 + _globals['_GETRUNDETAILSREQUEST']._serialized_end=3023 + _globals['_TASKRUNDETAIL']._serialized_start=3026 + _globals['_TASKRUNDETAIL']._serialized_end=3196 + _globals['_GETRUNDETAILSRESPONSE']._serialized_start=3199 + _globals['_GETRUNDETAILSRESPONSE']._serialized_end=3459 + _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_start=3393 + _globals['_GETRUNDETAILSRESPONSE_TASKRUNSENTRY']._serialized_end=3459 + _globals['_ADMINSERVICE']._serialized_start=3817 + _globals['_ADMINSERVICE']._serialized_end=4280 # @@protoc_insertion_point(module_scope) diff --git a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi index 3015f82678..837bc33b43 100644 --- a/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi +++ b/sdks/python/hatchet_sdk/contracts/v1/workflows_pb2.pyi @@ -107,7 +107,7 @@ class ReplayTasksResponse(_message.Message): def __init__(self, replayed_tasks: _Optional[_Iterable[str]] = ...) -> None: ... class TriggerWorkflowRunRequest(_message.Message): - __slots__ = ("workflow_name", "input", "additional_metadata", "priority", "desired_worker_labels") + __slots__ = ("workflow_name", "input", "additional_metadata", "priority", "desired_worker_labels", "idempotency_key") class DesiredWorkerLabelsEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -120,12 +120,14 @@ class TriggerWorkflowRunRequest(_message.Message): ADDITIONAL_METADATA_FIELD_NUMBER: _ClassVar[int] PRIORITY_FIELD_NUMBER: _ClassVar[int] DESIRED_WORKER_LABELS_FIELD_NUMBER: _ClassVar[int] + IDEMPOTENCY_KEY_FIELD_NUMBER: _ClassVar[int] workflow_name: str input: bytes additional_metadata: bytes priority: int desired_worker_labels: _containers.MessageMap[str, _trigger_pb2.DesiredWorkerLabels] - def __init__(self, workflow_name: _Optional[str] = ..., input: _Optional[bytes] = ..., additional_metadata: _Optional[bytes] = ..., priority: _Optional[int] = ..., desired_worker_labels: _Optional[_Mapping[str, _trigger_pb2.DesiredWorkerLabels]] = ...) -> None: ... + idempotency_key: str + def __init__(self, workflow_name: _Optional[str] = ..., input: _Optional[bytes] = ..., additional_metadata: _Optional[bytes] = ..., priority: _Optional[int] = ..., desired_worker_labels: _Optional[_Mapping[str, _trigger_pb2.DesiredWorkerLabels]] = ..., idempotency_key: _Optional[str] = ...) -> None: ... class TriggerWorkflowRunResponse(_message.Message): __slots__ = ("external_id",) diff --git a/sdks/typescript/generate-protoc.sh b/sdks/typescript/generate-protoc.sh index 39c8ea155f..b93d9fe1ca 100755 --- a/sdks/typescript/generate-protoc.sh +++ b/sdks/typescript/generate-protoc.sh @@ -1,3 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + # Directory to write generated code to (.js and .d.ts files) OUT_DIR="./src/protoc" @@ -8,12 +12,22 @@ else IN_DIR="../../api-contracts" fi -# Generate code -./node_modules/.bin/grpc_tools_node_protoc \ +# Generate code. +# Prefer system `protoc` (available in Nix environments) over `grpc-tools`' bundled protoc. +PROTOC_BIN="${PROTOC_BIN:-}" +if [ -z "$PROTOC_BIN" ]; then + if command -v protoc >/dev/null 2>&1; then + PROTOC_BIN="protoc" + else + PROTOC_BIN="./node_modules/.bin/grpc_tools_node_protoc" + fi +fi + +"$PROTOC_BIN" \ --plugin=protoc-gen-ts_proto=./node_modules/.bin/protoc-gen-ts_proto \ - --ts_proto_out=$OUT_DIR \ + --ts_proto_out="$OUT_DIR" \ --ts_proto_opt=outputServices=nice-grpc,outputServices=generic-definitions,useExactTypes=false \ - --proto_path=$IN_DIR \ - $IN_DIR/**/*.proto + --proto_path="$IN_DIR" \ + $(find "$IN_DIR" -type f -name '*.proto' -print) pnpm lint:fix diff --git a/sdks/typescript/src/clients/admin/admin-client.test.ts b/sdks/typescript/src/clients/admin/admin-client.test.ts index 67fc6d66eb..26d52a0eee 100644 --- a/sdks/typescript/src/clients/admin/admin-client.test.ts +++ b/sdks/typescript/src/clients/admin/admin-client.test.ts @@ -164,5 +164,47 @@ describe('AdminClient', () => { }) ); }); + + it('should forward idempotencyKey in batch requests', async () => { + const bulkSpy = jest.spyOn(client.client, 'bulkTriggerWorkflow').mockResolvedValue({ + workflowRunIds: ['run-1'], + }); + + await client.runWorkflows([ + { + workflowName: 'workflowName', + input: { hello: 'world' }, + options: { + idempotencyKey: 'idem-batch-1', + }, + }, + ]); + + expect(bulkSpy).toHaveBeenCalledWith( + expect.objectContaining({ + workflows: [ + expect.objectContaining({ + idempotencyKey: 'idem-batch-1', + }), + ], + }) + ); + }); + }); + + describe('runWorkflow', () => { + it('should forward idempotencyKey in single requests', async () => { + const triggerSpy = jest + .spyOn(client.client, 'triggerWorkflow') + .mockResolvedValue('run-1' as any); + + await client.runWorkflow('workflowName', { hello: 'world' }, { idempotencyKey: 'idem-1' }); + + expect(triggerSpy).toHaveBeenCalledWith( + expect.objectContaining({ + idempotencyKey: 'idem-1', + }) + ); + }); }); }); diff --git a/sdks/typescript/src/clients/admin/admin-client.ts b/sdks/typescript/src/clients/admin/admin-client.ts index 6d4183177b..b79282d87d 100644 --- a/sdks/typescript/src/clients/admin/admin-client.ts +++ b/sdks/typescript/src/clients/admin/admin-client.ts @@ -77,6 +77,7 @@ export type WorkflowRun = { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; }; }; @@ -196,6 +197,7 @@ export class AdminClient { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; } ) { @@ -228,6 +230,7 @@ export class AdminClient { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; desiredWorkerId?: string | undefined; priority?: Priority; @@ -291,6 +294,7 @@ export class AdminClient { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; desiredWorkerId?: string | undefined; priority?: Priority; diff --git a/sdks/typescript/src/protoc/dispatcher/dispatcher.ts b/sdks/typescript/src/protoc/dispatcher/dispatcher.ts index 4cac0ea433..317d737158 100644 --- a/sdks/typescript/src/protoc/dispatcher/dispatcher.ts +++ b/sdks/typescript/src/protoc/dispatcher/dispatcher.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: dispatcher/dispatcher.proto /* eslint-disable */ diff --git a/sdks/typescript/src/protoc/events/events.ts b/sdks/typescript/src/protoc/events/events.ts index 2d3a4cb1fd..65492ff8f2 100644 --- a/sdks/typescript/src/protoc/events/events.ts +++ b/sdks/typescript/src/protoc/events/events.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: events/events.proto /* eslint-disable */ diff --git a/sdks/typescript/src/protoc/google/protobuf/timestamp.ts b/sdks/typescript/src/protoc/google/protobuf/timestamp.ts index ee4b6339ba..036fe036ce 100644 --- a/sdks/typescript/src/protoc/google/protobuf/timestamp.ts +++ b/sdks/typescript/src/protoc/google/protobuf/timestamp.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: google/protobuf/timestamp.proto /* eslint-disable */ @@ -82,8 +82,8 @@ export const protobufPackage = 'google.protobuf'; * {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional * seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), * are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone - * is required. A proto3 JSON serializer should always use UTC (as indicated by - * "Z") when printing the Timestamp type and a proto3 JSON parser should be + * is required. A ProtoJSON serializer should always use UTC (as indicated by + * "Z") when printing the Timestamp type and a ProtoJSON parser should be * able to accept both UTC and other timezones (as indicated by an offset). * * For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past @@ -97,20 +97,21 @@ export const protobufPackage = 'google.protobuf'; * [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with * the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use * the Joda Time's [`ISODateTimeFormat.dateTime()`]( - * http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D + * http://joda-time.sourceforge.net/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime() * ) to obtain a formatter capable of generating timestamps in this format. */ export interface Timestamp { /** - * Represents seconds of UTC time since Unix epoch - * 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - * 9999-12-31T23:59:59Z inclusive. + * Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must + * be between -62135596800 and 253402300799 inclusive (which corresponds to + * 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z). */ seconds: number; /** - * Non-negative fractions of a second at nanosecond resolution. Negative - * second values with fractions must still have non-negative nanos values - * that count forward in time. Must be from 0 to 999,999,999 + * Non-negative fractions of a second at nanosecond resolution. This field is + * the nanosecond portion of the duration, not an alternative to seconds. + * Negative second values with fractions must still have non-negative nanos + * values that count forward in time. Must be between 0 and 999,999,999 * inclusive. */ nanos: number; diff --git a/sdks/typescript/src/protoc/v1/dispatcher.ts b/sdks/typescript/src/protoc/v1/dispatcher.ts index 29545fe45f..675c27838f 100644 --- a/sdks/typescript/src/protoc/v1/dispatcher.ts +++ b/sdks/typescript/src/protoc/v1/dispatcher.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: v1/dispatcher.proto /* eslint-disable */ diff --git a/sdks/typescript/src/protoc/v1/shared/condition.ts b/sdks/typescript/src/protoc/v1/shared/condition.ts index ae8ce105c0..eb1584b551 100644 --- a/sdks/typescript/src/protoc/v1/shared/condition.ts +++ b/sdks/typescript/src/protoc/v1/shared/condition.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: v1/shared/condition.proto /* eslint-disable */ diff --git a/sdks/typescript/src/protoc/v1/shared/trigger.ts b/sdks/typescript/src/protoc/v1/shared/trigger.ts index 6b3a49268d..f0f5ebca9d 100644 --- a/sdks/typescript/src/protoc/v1/shared/trigger.ts +++ b/sdks/typescript/src/protoc/v1/shared/trigger.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: v1/shared/trigger.proto /* eslint-disable */ @@ -120,6 +120,8 @@ export interface TriggerWorkflowRequest { priority?: number | undefined; /** (optional) the desired worker labels for the workflow run, which will be used to determine which workers can pick up the workflow's tasks. if not set, defaults to an empty set of labels, which means any worker can pick up the tasks. */ desiredWorkerLabels: { [key: string]: DesiredWorkerLabels }; + /** (optional) idempotency key for deduplicating workflow runs */ + idempotencyKey?: string | undefined; } export interface TriggerWorkflowRequest_DesiredWorkerLabelsEntry { @@ -279,6 +281,7 @@ function createBaseTriggerWorkflowRequest(): TriggerWorkflowRequest { desiredWorkerId: undefined, priority: undefined, desiredWorkerLabels: {}, + idempotencyKey: undefined, }; } @@ -319,6 +322,9 @@ export const TriggerWorkflowRequest: MessageFns = { ).join(); } ); + if (message.idempotencyKey !== undefined) { + writer.uint32(90).string(message.idempotencyKey); + } return writer; }, @@ -415,6 +421,14 @@ export const TriggerWorkflowRequest: MessageFns = { } continue; } + case 11: { + if (tag !== 90) { + break; + } + + message.idempotencyKey = reader.string(); + continue; + } } if ((tag & 7) === 4 || tag === 0) { break; @@ -476,6 +490,11 @@ export const TriggerWorkflowRequest: MessageFns = { {} ) : {}, + idempotencyKey: isSet(object.idempotencyKey) + ? globalThis.String(object.idempotencyKey) + : isSet(object.idempotency_key) + ? globalThis.String(object.idempotency_key) + : undefined, }; }, @@ -520,6 +539,9 @@ export const TriggerWorkflowRequest: MessageFns = { }); } } + if (message.idempotencyKey !== undefined) { + obj.idempotencyKey = message.idempotencyKey; + } return obj; }, @@ -551,6 +573,7 @@ export const TriggerWorkflowRequest: MessageFns = { }, {} ); + message.idempotencyKey = object.idempotencyKey ?? undefined; return message; }, }; diff --git a/sdks/typescript/src/protoc/v1/workflows.ts b/sdks/typescript/src/protoc/v1/workflows.ts index 6e729c1514..56419f7f9f 100644 --- a/sdks/typescript/src/protoc/v1/workflows.ts +++ b/sdks/typescript/src/protoc/v1/workflows.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: v1/workflows.proto /* eslint-disable */ @@ -253,6 +253,8 @@ export interface TriggerWorkflowRunRequest { additionalMetadata: Uint8Array; priority?: number | undefined; desiredWorkerLabels: { [key: string]: DesiredWorkerLabels }; + /** (optional) idempotency key for deduplicating workflow runs */ + idempotencyKey?: string | undefined; } export interface TriggerWorkflowRunRequest_DesiredWorkerLabelsEntry { @@ -878,6 +880,7 @@ function createBaseTriggerWorkflowRunRequest(): TriggerWorkflowRunRequest { additionalMetadata: new Uint8Array(0), priority: undefined, desiredWorkerLabels: {}, + idempotencyKey: undefined, }; } @@ -906,6 +909,9 @@ export const TriggerWorkflowRunRequest: MessageFns = ).join(); } ); + if (message.idempotencyKey !== undefined) { + writer.uint32(50).string(message.idempotencyKey); + } return writer; }, @@ -962,6 +968,14 @@ export const TriggerWorkflowRunRequest: MessageFns = } continue; } + case 6: { + if (tag !== 50) { + break; + } + + message.idempotencyKey = reader.string(); + continue; + } } if ((tag & 7) === 4 || tag === 0) { break; @@ -1002,6 +1016,11 @@ export const TriggerWorkflowRunRequest: MessageFns = {} ) : {}, + idempotencyKey: isSet(object.idempotencyKey) + ? globalThis.String(object.idempotencyKey) + : isSet(object.idempotency_key) + ? globalThis.String(object.idempotency_key) + : undefined, }; }, @@ -1031,6 +1050,9 @@ export const TriggerWorkflowRunRequest: MessageFns = }); } } + if (message.idempotencyKey !== undefined) { + obj.idempotencyKey = message.idempotencyKey; + } return obj; }, @@ -1057,6 +1079,7 @@ export const TriggerWorkflowRunRequest: MessageFns = }, {} ); + message.idempotencyKey = object.idempotencyKey ?? undefined; return message; }, }; diff --git a/sdks/typescript/src/protoc/workflows/workflows.ts b/sdks/typescript/src/protoc/workflows/workflows.ts index 4dfdce3ef0..52a3f989a1 100644 --- a/sdks/typescript/src/protoc/workflows/workflows.ts +++ b/sdks/typescript/src/protoc/workflows/workflows.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: -// protoc-gen-ts_proto v2.11.8 -// protoc v3.19.1 +// protoc-gen-ts_proto v2.11.10 +// protoc v7.34.1 // source: workflows/workflows.proto /* eslint-disable */ diff --git a/sdks/typescript/src/v1/client/admin.test.ts b/sdks/typescript/src/v1/client/admin.test.ts index c3439c38f1..8336264bf3 100644 --- a/sdks/typescript/src/v1/client/admin.test.ts +++ b/sdks/typescript/src/v1/client/admin.test.ts @@ -56,6 +56,15 @@ describe('AdminClient workflow name normalization', () => { ); }); + it('runWorkflow forwards idempotencyKey', async () => { + const admin = createMockAdmin(); + await admin.runWorkflow('my-workflow', {}, { idempotencyKey: 'idem-1' }); + + expect(admin.workflowsGrpc.triggerWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'idem-1' }) + ); + }); + it('runWorkflows lowercases workflow names in batch', async () => { const admin = createMockAdmin(); await admin.runWorkflows([ @@ -72,4 +81,29 @@ describe('AdminClient workflow name normalization', () => { }) ); }); + + it('runWorkflows forwards idempotencyKey in batch', async () => { + const admin = createMockAdmin(); + await admin.runWorkflows([ + { + workflowName: 'WorkflowOne', + input: {}, + options: { idempotencyKey: 'idem-batch-1' }, + }, + { + workflowName: 'WorkflowTwo', + input: {}, + options: { idempotencyKey: 'idem-batch-2' }, + }, + ]); + + expect(admin.workflowsGrpc.bulkTriggerWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ + workflows: expect.arrayContaining([ + expect.objectContaining({ idempotencyKey: 'idem-batch-1' }), + expect.objectContaining({ idempotencyKey: 'idem-batch-2' }), + ]), + }) + ); + }); }); diff --git a/sdks/typescript/src/v1/client/admin.ts b/sdks/typescript/src/v1/client/admin.ts index 87540053cc..ce44410cdd 100644 --- a/sdks/typescript/src/v1/client/admin.ts +++ b/sdks/typescript/src/v1/client/admin.ts @@ -64,6 +64,7 @@ export type WorkflowRun = { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; }; }; @@ -128,6 +129,7 @@ export class AdminClient { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; desiredWorkerId?: string | undefined; priority?: Priority; @@ -212,6 +214,7 @@ export class AdminClient { parentStepRunId?: string | undefined; childIndex?: number | undefined; childKey?: string | undefined; + idempotencyKey?: string | undefined; additionalMetadata?: Record | undefined; desiredWorkerId?: string | undefined; priority?: Priority; diff --git a/sdks/typescript/src/v1/declaration.ts b/sdks/typescript/src/v1/declaration.ts index 24d19d507e..361b4ff62c 100644 --- a/sdks/typescript/src/v1/declaration.ts +++ b/sdks/typescript/src/v1/declaration.ts @@ -75,6 +75,11 @@ export type RunOpts = { */ priority?: Priority; + /** + * (optional) idempotency key for deduplicating workflow runs. + */ + idempotencyKey?: string; + /** * (optional) if the task run should be run on the same worker. * only used if spawned from within a parent task. diff --git a/sql/schema/v1-core.sql b/sql/schema/v1-core.sql index e75bf4ab6d..b2444b5827 100644 --- a/sql/schema/v1-core.sql +++ b/sql/schema/v1-core.sql @@ -2283,6 +2283,7 @@ CREATE TABLE v1_idempotency_key ( expires_at TIMESTAMPTZ NOT NULL, claimed_by_external_id UUID, + last_denied_at TIMESTAMPTZ, inserted_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,